From 225760855ece6de08a24c5a2301f87ba7a4bc649 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Fri, 17 Jul 2026 14:30:31 +0100 Subject: [PATCH 01/20] Add vLLM Offline Backend for batch processing Introduces VLLMOfflineBackend inheriting from VLLMPythonBackend, using vLLM's synchronous LLM engine for micro-batched inference. Wires RequestStateStats metrics into response timings. Includes registration test, docs, and backends.md update. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Maryam Tahhan --- docs/guides/backends.md | 4 + docs/guides/vllm-offline-backend.md | 64 +++ src/guidellm/backends/vllm_python/__init__.py | 3 +- src/guidellm/backends/vllm_python/offline.py | 531 ++++++++++++++++++ tests/unit/backends/test_backend.py | 19 + 5 files changed, 620 insertions(+), 1 deletion(-) create mode 100644 docs/guides/vllm-offline-backend.md create mode 100644 src/guidellm/backends/vllm_python/offline.py diff --git a/docs/guides/backends.md b/docs/guides/backends.md index f810420c1..13d54fb61 100644 --- a/docs/guides/backends.md +++ b/docs/guides/backends.md @@ -32,6 +32,10 @@ GuideLLM supports OpenAI-compatible HTTP servers, which provide a standardized A GuideLLM supports running inference in the same process using the **vLLM Python backend** (`vllm_python`). This backend runs inference in the same process as GuideLLM's using vLLM's python API (AsyncLLMEngine), without an HTTP server. For setup, installation options (container, existing vLLM, pip), and examples, see [vLLM Python backend](vllm-python-backend.md). +### vLLM Offline Backend + +The **vLLM offline backend** (`vllm_offline`) uses vLLM's synchronous `LLM` engine for batch-oriented inference. Requests are queued and dispatched in configurable batches, removing per-request scheduling overhead. This is ideal for throughput benchmarking. For setup and examples, see [vLLM Offline backend](vllm-offline-backend.md). + ## Examples for Spinning Up Compatible Servers ### 1. vLLM diff --git a/docs/guides/vllm-offline-backend.md b/docs/guides/vllm-offline-backend.md new file mode 100644 index 000000000..5c2b8d79e --- /dev/null +++ b/docs/guides/vllm-offline-backend.md @@ -0,0 +1,64 @@ +# vLLM Offline Backend + +The **vLLM offline backend** (`vllm_offline`) runs batch inference in the same process as GuideLLM using vLLM's synchronous [LLM](https://docs.vllm.ai/) engine. Requests are queued and dispatched in configurable batches via `LLM.generate()`, removing per-request scheduling overhead. This is ideal for throughput benchmarking where latency per individual request is less important than aggregate throughput. + +Like the `vllm_python` backend, no HTTP server is involved. You do **not** pass a `target`; you **must** pass `model` in the backend configuration. + +For all engine options and supported models, see vLLM's [Engine Arguments](https://docs.vllm.ai/en/stable/configuration/engine_args/) and the [vLLM documentation](https://docs.vllm.ai/). + +## Installation + +Installation is the same as for the [vLLM Python backend](vllm-python-backend.md#installation). The offline backend uses the same vLLM package. + +## Basic example + +Run a benchmark with the vLLM offline backend: + +```bash +guidellm run \ + --backend kind=vllm_offline,model=Qwen/Qwen3-0.6B,batch_size=32 \ + --data kind=synthetic_text,prompt_tokens=256,output_tokens=128 \ + --profile kind=constant,rate=3 \ + --constraint kind=max_duration,seconds=20 +``` + +## Offline vs Python backend + +| Feature | `vllm_python` | `vllm_offline` | +|---|---|---| +| Engine | `AsyncLLMEngine` (async) | `LLM` (synchronous, batch) | +| Streaming | Supported | Not supported | +| Batching | Per-request async scheduling | Configurable micro-batches | +| Best for | Latency profiling, streaming | Throughput benchmarking | + +## Backend options + +- **`batch_size`** (default: `32`)\ + Maximum number of requests to accumulate before dispatching a single `LLM.generate()` call. Larger batches amortize engine overhead but increase per-request latency. + +- **`model`** (required)\ + Hugging Face model identifier or filesystem path for vLLM to load. + +- **`request_format`**\ + Controls how chat prompts are built. Same options as `vllm_python`: `plain`, `default-template`, or a Jinja2 template path/string. + +- **`vllm_config`**\ + Engine options passed as a nested dict. Uses vLLM's `EngineArgs` parameter names (Python form, not CLI form). See the [vLLM Python backend docs](vllm-python-backend.md#request-format-and-backend-options) for details on `vllm_config`. + + Example with JSON: + + ```bash + --backend '{"kind":"vllm_offline","model":"Qwen/Qwen3-0.6B","batch_size":64,"vllm_config":{"gpu_memory_utilization":0.8,"max_model_len":4096}}' + ``` + +> [!IMPORTANT] +> +> The `model` field in the backend configuration is required for `vllm_offline`. If `model` is also set inside `vllm_config`, the top-level `model` field takes precedence. + +## See also + +- [vLLM Python Backend](vllm-python-backend.md) -- Async per-request backend. +- [Backends](backends.md) -- Overview of supported backends. +- [Run a benchmark](../getting-started/benchmark.md) -- General benchmark options. +- [vLLM Engine Arguments](https://docs.vllm.ai/en/stable/configuration/engine_args/) -- CLI-oriented docs; use Python names in `vllm_config`. +- [vLLM documentation](https://docs.vllm.ai/) diff --git a/src/guidellm/backends/vllm_python/__init__.py b/src/guidellm/backends/vllm_python/__init__.py index fb8f4703b..a6851a2f5 100644 --- a/src/guidellm/backends/vllm_python/__init__.py +++ b/src/guidellm/backends/vllm_python/__init__.py @@ -5,7 +5,8 @@ GenerationResponse from vLLM output. """ +from .offline import VLLMOfflineBackend from .vllm import VLLMPythonBackend from .vllm_response import VLLMResponseHandler -__all__ = ["VLLMPythonBackend", "VLLMResponseHandler"] +__all__ = ["VLLMOfflineBackend", "VLLMPythonBackend", "VLLMResponseHandler"] diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py new file mode 100644 index 000000000..8eb8554af --- /dev/null +++ b/src/guidellm/backends/vllm_python/offline.py @@ -0,0 +1,531 @@ +""" +VLLM Offline (batch) backend implementation for GuideLLM. + +Provides batch-oriented inference using vLLM's synchronous LLM engine. +Requests are queued and processed in configurable batches, removing the +overhead of per-request engine interaction while still integrating with +the standard GuideLLM scheduler lifecycle. +""" + +from __future__ import annotations + +import asyncio +import time +import uuid +from collections.abc import AsyncIterator +from dataclasses import dataclass, field +from typing import Any, Literal, cast + +from pydantic import ConfigDict, Field, PositiveInt, model_validator + +from guidellm.backends.backend import Backend, BackendArgs +from guidellm.backends.vllm_python.vllm import ( + _CHAT_TEMPLATE_UNSET, + VLLMPythonBackend, + VLLMPythonBackendArgs, +) +from guidellm.backends.vllm_python.vllm_response import VLLMResponseHandler +from guidellm.extras import vllm +from guidellm.logger import logger +from guidellm.schemas import ( + GenerationRequest, + GenerationResponse, + RequestInfo, + StandardBaseModel, +) + +__all__ = ["VLLMOfflineBackend", "VLLMOfflineBackendArgs"] + + +@BackendArgs.register("vllm_offline") +class VLLMOfflineBackendArgs(VLLMPythonBackendArgs): + """Pydantic model for VLLM Offline backend creation arguments. + + Extends :class:`VLLMPythonBackendArgs` with batch-specific options + and removes the ``stream`` field (offline generation is always + non-streaming). + """ + + kind: Literal["vllm_offline"] = Field( # type: ignore[assignment] + default="vllm_offline", + description="Backend type identifier for VLLM Offline backend.", + ) + batch_size: PositiveInt = Field( + default=32, + description=( + "Maximum number of requests to accumulate before " + "dispatching a single vLLM generate() call." + ), + ) + + # Hide the inherited ``stream`` field -- offline is never streaming. + stream: Literal[False] = Field( # type: ignore[assignment] + default=False, + exclude=True, + description="Offline backend does not support streaming.", + ) + + @model_validator(mode="after") + def validate_vllm_config(self): + """Set defaults on vllm_config and ensure model is set.""" + if "model" in self.vllm_config: + logger.warning( + "The `model` input was passed to the vllm offline " + "backend with the `vllm_config` input. Ignoring and " + "overwriting with the value from the `model` input." + ) + self.vllm_config["model"] = self.model + return self + + +class _OfflineResolvedRequest(StandardBaseModel): + """Fully resolved request for the offline backend. + + Same as the online ``_ResolvedRequest`` but without a ``stream`` + field, since offline generation never streams. + """ + + model_config = ConfigDict(frozen=True) + + prompt: str = Field( + description="Fully resolved prompt string.", + ) + multi_modal_data: dict[str, Any] | None = Field( + default=None, + description=("vLLM multi_modal_data from image/audio/video columns."), + ) + + +@dataclass +class _BatchedRequest: + """Tracks a single request waiting for batch processing.""" + + request: GenerationRequest + request_info: RequestInfo + resolved_prompt: str + multi_modal_data: dict[str, Any] | None + max_tokens: int | None + request_id: str = field(default_factory=lambda: str(uuid.uuid4())) + result: Any = None # vllm.RequestOutput once available + ready: asyncio.Event = field(default_factory=asyncio.Event) + + +@Backend.register("vllm_offline") +class VLLMOfflineBackend(VLLMPythonBackend): + """ + Batch-oriented Python API backend for VLLM inference engine. + + Queues incoming requests and dispatches them in batches via + ``vllm.LLM.generate()``, which runs the synchronous offline + engine. This avoids per-request scheduling overhead and is + ideal for throughput benchmarking. + + Example: + :: + backend = VLLMOfflineBackend( + VLLMOfflineBackendArgs( + model="meta-llama/Llama-2-7b-chat-hf", + batch_size=16, + ) + ) + await backend.process_startup() + async for response, request_info in backend.resolve( + request, info + ): + process_response(response) + await backend.process_shutdown() + """ + + _args: VLLMOfflineBackendArgs + + @classmethod + def backend_args(cls) -> type[BackendArgs]: + """Return the Pydantic model for this backend's creation + arguments. + """ + return VLLMOfflineBackendArgs + + def __init__( + self, + arguments: VLLMOfflineBackendArgs, + ): + """ + Initialize VLLM Offline backend. + + Sets up batch processing state in addition to the base + backend initialisation. + """ + super().__init__(arguments) + + # Batch processing state + self._batch_lock = asyncio.Lock() + self._pending_batch: list[_BatchedRequest] = [] + self._processing_task: asyncio.Task[None] | None = None + self._shutting_down = False + + # The synchronous vLLM LLM engine (set during startup) + self._llm: Any = None # vllm.LLM + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + async def process_startup(self): + """ + Initialize the synchronous vLLM LLM engine. + + Runs engine construction in the default executor so the + event loop is not blocked during model loading. + + :raises RuntimeError: If backend is already initialised + """ + if self._in_process: + raise RuntimeError("Backend already started up for process.") + + loop = asyncio.get_running_loop() + config = dict(self._args.vllm_config) + + engine_args = vllm.EngineArgs( # type: ignore[attr-defined] + **config, + ) + self._llm = await loop.run_in_executor( + None, + vllm.LLM.from_engine_args, # type: ignore[attr-defined] + engine_args, + ) + self._in_process = True + self._shutting_down = False + + async def process_shutdown(self): + """ + Drain pending batch and tear down the vLLM LLM engine. + + :raises RuntimeError: If backend was not properly initialised + """ + if not self._in_process: + raise RuntimeError("Backend not started up for process.") + + self._shutting_down = True + + # Drain any remaining requests + async with self._batch_lock: + if self._pending_batch: + await self._process_batch() + + if self._processing_task is not None: + self._processing_task.cancel() + self._processing_task = None + + if self._llm is not None: + del self._llm + self._llm = None + + self._in_process = False + + async def validate(self): + """ + Validate backend readiness. + + The synchronous LLM engine has no ``check_health`` method, + so we simply verify the engine was initialised. + + :raises RuntimeError: If backend is not initialised + """ + self._validate_backend_initialized() + + def _validate_backend_initialized(self) -> Any: # type: ignore[override] + """ + Validate that the backend is initialised and return the LLM. + + :raises RuntimeError: If backend is not initialised + :return: The initialised ``vllm.LLM`` instance + """ + if self._llm is None: + raise RuntimeError("Backend not started up for process.") + return self._llm + + # ------------------------------------------------------------------ + # Chat template / tokenizer (overrides for offline tokenizer path) + # ------------------------------------------------------------------ + + def _extract_prompt_chat_tokenizer( + self, formatted_messages: list[dict[str, Any]] + ) -> str: + """Apply tokenizer chat template to formatted messages. + + Accesses the tokenizer through the synchronous LLM engine + path (``llm.llm_engine.tokenizer.tokenizer``) instead of + ``engine.tokenizer`` used by the async parent. + """ + llm = self._validate_backend_initialized() + tokenizer = llm.llm_engine.tokenizer.tokenizer + if tokenizer is None: + raise RuntimeError("Backend engine has no tokenizer.") + + if self._args.request_format in ( + "plain", + "default-template", + ): + resolved: str | None = None + else: + if self._resolved_chat_template is _CHAT_TEMPLATE_UNSET: + self._resolved_chat_template = self._resolve_chat_template() + resolved = cast( + "str | None", + self._resolved_chat_template, + ) + + if resolved is not None: + tokenizer.chat_template = resolved + + prompt = tokenizer.apply_chat_template( + formatted_messages, + tokenize=False, + add_generation_prompt=True, + ) + if isinstance(prompt, str): + return prompt + raise RuntimeError("Backend received unexpected type from tokenizer.") + + # ------------------------------------------------------------------ + # Request resolution (no ``stream`` field) + # ------------------------------------------------------------------ + + def _resolve_request( # type: ignore[override] + self, request: GenerationRequest + ) -> _OfflineResolvedRequest: + """ + Build a fully resolved request for offline generation. + + Delegates to the parent's resolution logic but returns an + ``_OfflineResolvedRequest`` (without a ``stream`` field). + + :param request: Column-based generation request + :return: Resolved request with formatted prompt and + multimodal data + """ + parent_resolved = super()._resolve_request(request) + return _OfflineResolvedRequest( + prompt=parent_resolved.prompt, + multi_modal_data=parent_resolved.multi_modal_data, + ) + + # ------------------------------------------------------------------ + # Batch processing + # ------------------------------------------------------------------ + + async def _process_batch(self) -> None: + """Collect all pending requests and run ``LLM.generate()``. + + Must be called while holding ``_batch_lock``. Results are + distributed back to callers via each ``_BatchedRequest.ready`` + event. + """ + if not self._pending_batch: + return + + batch = list(self._pending_batch) + self._pending_batch.clear() + + # Build per-request generate inputs + prompts: list[str | dict[str, Any]] = [] + sampling_params_list: list[Any] = [] + for batched_req in batch: + if batched_req.multi_modal_data: + prompts.append( + { + "prompt": batched_req.resolved_prompt, + "multi_modal_data": (batched_req.multi_modal_data), + } + ) + else: + prompts.append(batched_req.resolved_prompt) + + sampling_params_list.append( + self._create_sampling_params( + batched_req.max_tokens, + ) + ) + + loop = asyncio.get_running_loop() + + try: + outputs = await loop.run_in_executor( + None, + lambda: self._llm.generate( # type: ignore[union-attr] + prompts, + sampling_params=sampling_params_list, + use_tqdm=False, + ), + ) + except (RuntimeError, ValueError, TypeError, OSError, KeyError) as exc: + logger.error( + "vLLM offline generate() failed: {}: {}", + type(exc).__name__, + exc, + ) + # Signal all waiters so they can raise + for batched_req in batch: + batched_req.result = exc + batched_req.ready.set() + return + + # Distribute results back to callers + for batched_req, output in zip(batch, outputs, strict=True): + batched_req.result = output + batched_req.ready.set() + + async def _maybe_process_batch(self) -> None: + """Trigger batch processing if the batch is full.""" + async with self._batch_lock: + if len(self._pending_batch) >= self._args.batch_size: + await self._process_batch() + + async def resolve( # type: ignore[override, misc] + self, + request: GenerationRequest, + request_info: RequestInfo, + history: ( + list[tuple[GenerationRequest, GenerationResponse]] | None + ) = None, + ) -> AsyncIterator[tuple[GenerationResponse, RequestInfo]]: + """ + Queue a request for batch processing and yield the response. + + Resolves the request (chat template, placeholders, multimodal + data), adds it to the pending batch, and waits until the + batch has been processed. The caller receives exactly one + ``(response, request_info)`` pair. + + :param request: Generation request with content and params + :param request_info: Request tracking info updated with + timing metadata + :param history: Conversation history (not supported) + :raises NotImplementedError: If history is provided + :raises RuntimeError: If backend is not initialised or + generation fails + :yields: Single tuple of (response, updated_request_info) + """ + self._validate_backend_initialized() + self._validate_history(history) + + resolved = self._resolve_request(request) + + max_tokens = ( + request.output_metrics.text_tokens + if request.output_metrics.text_tokens + else None + ) + + batched_req = _BatchedRequest( + request=request, + request_info=request_info, + resolved_prompt=resolved.prompt, + multi_modal_data=resolved.multi_modal_data, + max_tokens=max_tokens, + ) + + request_info.timings.request_start = time.time() + + # Enqueue + async with self._batch_lock: + self._pending_batch.append(batched_req) + + # If the batch is full, process immediately + await self._maybe_process_batch() + + # If not full yet, schedule a deferred flush so the last + # partial batch does not sit forever. + if not batched_req.ready.is_set(): + await self._schedule_deferred_flush() + + # Wait for this request's result + await batched_req.ready.wait() + + result = batched_req.result + + # Propagate generation errors + if isinstance(result, BaseException): + self._raise_generation_error(result) + + request_output = result # vllm.RequestOutput + + # Wire vLLM request metrics into timing info + self._wire_vllm_metrics(request_info, request_output) + + request_info.timings.request_end = time.time() + + text = self._text_from_output(request_output) + usage = self._usage_from_output(request_output) + response_id = ( + request_output.request_id + if request_output.request_id + else None + ) + + response = VLLMResponseHandler.build_response( + request, text, usage, response_id=response_id + ) + yield response, request_info + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + async def _schedule_deferred_flush(self) -> None: + """Schedule a task that flushes the pending batch after a + short delay, ensuring partial batches are not stuck. + """ + + async def _deferred_flush() -> None: + # Yield control so other requests can arrive + await asyncio.sleep(0) + async with self._batch_lock: + await self._process_batch() + + # Only schedule one deferred flush at a time + if self._processing_task is None or self._processing_task.done(): + self._processing_task = asyncio.ensure_future(_deferred_flush()) + + @staticmethod + def _wire_vllm_metrics( + request_info: RequestInfo, + request_output: Any, + ) -> None: + """Populate request timing from vLLM ``RequestOutput.metrics``. + + The ``metrics`` attribute (a ``RequestStateStats``) carries + engine-internal timestamps for scheduling, first-token, and + completion. We map those onto the ``RequestInfo.timings`` + fields so downstream analysis has per-request granularity. + + ``request_start`` / ``request_end`` are left as user-observed + wall-clock times and are **not** overwritten here. + """ + metrics = getattr(request_output, "metrics", None) + if metrics is None: + return + + first_token_ts = getattr(metrics, "first_token_ts", None) + last_token_ts = getattr(metrics, "last_token_ts", None) + + if first_token_ts is not None and first_token_ts > 0: + request_info.timings.first_token_iteration = first_token_ts + request_info.timings.first_request_iteration = first_token_ts + + if last_token_ts is not None and last_token_ts > 0: + request_info.timings.last_token_iteration = last_token_ts + request_info.timings.last_request_iteration = last_token_ts + + num_gen = getattr(metrics, "num_generation_tokens", 0) + if num_gen > 0: + request_info.timings.token_iterations = num_gen + request_info.timings.request_iterations = 1 + elif ( + request_output.outputs + and request_output.outputs[0].token_ids is not None + ): + request_info.timings.token_iterations = len( + request_output.outputs[0].token_ids + ) + request_info.timings.request_iterations = 1 diff --git a/tests/unit/backends/test_backend.py b/tests/unit/backends/test_backend.py index 88c7d6697..09c0fb4b1 100644 --- a/tests/unit/backends/test_backend.py +++ b/tests/unit/backends/test_backend.py @@ -18,6 +18,10 @@ OpenAIWebSocketBackendArgs, ) from guidellm.backends.openai.http import OpenAIHTTPBackendArgs +from guidellm.backends.vllm_python.offline import ( + VLLMOfflineBackend, + VLLMOfflineBackendArgs, +) from guidellm.backends.vllm_python.vllm import ( VLLMPythonBackend, VLLMPythonBackendArgs, @@ -558,6 +562,21 @@ async def default_model(self): assert isinstance(backend, TestDecoratorBackend) assert backend.info == {"test_param": "custom"} + @pytest.mark.smoke + def test_vllm_offline_backend_registered(self): + """ + Test that vllm_offline backend is registered and createable. + ## WRITTEN BY AI ## + """ + assert Backend.is_registered("vllm_offline") + args = VLLMOfflineBackendArgs(model="test-model") + backend = Backend.create(args) + assert isinstance(backend, VLLMOfflineBackend) + assert isinstance(backend, VLLMPythonBackend) + assert backend._args.model == "test-model" + assert backend._args.batch_size == 32 + assert backend.kind == "vllm_offline" + @pytest.mark.smoke def test_registered_objects(self): """Test Backend.registered_objects method returns registered backends.""" From ba439f6a0910bce3391ecb151a0b33163a48ed57 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Fri, 17 Jul 2026 15:33:26 +0100 Subject: [PATCH 02/20] Fix tokenizer access path for vLLM V1 LLM engine Use llm.get_tokenizer() instead of llm.llm_engine.tokenizer.tokenizer which doesn't exist in the V1 engine. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Maryam Tahhan --- src/guidellm/backends/vllm_python/offline.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index 8eb8554af..5318b0620 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -253,12 +253,11 @@ def _extract_prompt_chat_tokenizer( ) -> str: """Apply tokenizer chat template to formatted messages. - Accesses the tokenizer through the synchronous LLM engine - path (``llm.llm_engine.tokenizer.tokenizer``) instead of - ``engine.tokenizer`` used by the async parent. + Accesses the tokenizer through ``llm.get_tokenizer()`` + instead of ``engine.tokenizer`` used by the async parent. """ llm = self._validate_backend_initialized() - tokenizer = llm.llm_engine.tokenizer.tokenizer + tokenizer = llm.get_tokenizer() if tokenizer is None: raise RuntimeError("Backend engine has no tokenizer.") From 5467ec751f78724b6d66686c01cf26d780273567 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Fri, 17 Jul 2026 15:41:25 +0100 Subject: [PATCH 03/20] Fix formatting, metrics, and mdformat issues - Apply ruff format to offline.py - Fix _wire_vllm_metrics: only use token counts, not monotonic timestamps that produce garbage when mixed with wall-clock times - Apply mdformat to markdown docs Co-Authored-By: Claude Opus 4.6 Signed-off-by: Maryam Tahhan --- docs/guides/vllm-offline-backend.md | 12 ++--- src/guidellm/backends/vllm_python/offline.py | 47 +++++--------------- 2 files changed, 18 insertions(+), 41 deletions(-) diff --git a/docs/guides/vllm-offline-backend.md b/docs/guides/vllm-offline-backend.md index 5c2b8d79e..9f4f7ba78 100644 --- a/docs/guides/vllm-offline-backend.md +++ b/docs/guides/vllm-offline-backend.md @@ -24,12 +24,12 @@ guidellm run \ ## Offline vs Python backend -| Feature | `vllm_python` | `vllm_offline` | -|---|---|---| -| Engine | `AsyncLLMEngine` (async) | `LLM` (synchronous, batch) | -| Streaming | Supported | Not supported | -| Batching | Per-request async scheduling | Configurable micro-batches | -| Best for | Latency profiling, streaming | Throughput benchmarking | +| Feature | `vllm_python` | `vllm_offline` | +| --------- | ---------------------------- | -------------------------- | +| Engine | `AsyncLLMEngine` (async) | `LLM` (synchronous, batch) | +| Streaming | Supported | Not supported | +| Batching | Per-request async scheduling | Configurable micro-batches | +| Best for | Latency profiling, streaming | Throughput benchmarking | ## Backend options diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index 5318b0620..defc6a974 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -384,9 +384,7 @@ async def resolve( # type: ignore[override, misc] self, request: GenerationRequest, request_info: RequestInfo, - history: ( - list[tuple[GenerationRequest, GenerationResponse]] | None - ) = None, + history: (list[tuple[GenerationRequest, GenerationResponse]] | None) = None, ) -> AsyncIterator[tuple[GenerationResponse, RequestInfo]]: """ Queue a request for batch processing and yield the response. @@ -456,11 +454,7 @@ async def resolve( # type: ignore[override, misc] text = self._text_from_output(request_output) usage = self._usage_from_output(request_output) - response_id = ( - request_output.request_id - if request_output.request_id - else None - ) + response_id = request_output.request_id if request_output.request_id else None response = VLLMResponseHandler.build_response( request, text, usage, response_id=response_id @@ -491,39 +485,22 @@ def _wire_vllm_metrics( request_info: RequestInfo, request_output: Any, ) -> None: - """Populate request timing from vLLM ``RequestOutput.metrics``. - - The ``metrics`` attribute (a ``RequestStateStats``) carries - engine-internal timestamps for scheduling, first-token, and - completion. We map those onto the ``RequestInfo.timings`` - fields so downstream analysis has per-request granularity. - - ``request_start`` / ``request_end`` are left as user-observed - wall-clock times and are **not** overwritten here. + """Populate iteration counts from vLLM ``RequestOutput``. + + vLLM's ``RequestStateStats`` timestamps are monotonic-clock + values that cannot be mixed with the wall-clock + ``request_start`` / ``request_end``, so we only extract + token counts here. Iteration timing fields are left to the + wall-clock ``request_start`` / ``request_end`` already set + by the caller. """ metrics = getattr(request_output, "metrics", None) - if metrics is None: - return + num_gen = getattr(metrics, "num_generation_tokens", 0) if metrics else 0 - first_token_ts = getattr(metrics, "first_token_ts", None) - last_token_ts = getattr(metrics, "last_token_ts", None) - - if first_token_ts is not None and first_token_ts > 0: - request_info.timings.first_token_iteration = first_token_ts - request_info.timings.first_request_iteration = first_token_ts - - if last_token_ts is not None and last_token_ts > 0: - request_info.timings.last_token_iteration = last_token_ts - request_info.timings.last_request_iteration = last_token_ts - - num_gen = getattr(metrics, "num_generation_tokens", 0) if num_gen > 0: request_info.timings.token_iterations = num_gen request_info.timings.request_iterations = 1 - elif ( - request_output.outputs - and request_output.outputs[0].token_ids is not None - ): + elif request_output.outputs and request_output.outputs[0].token_ids is not None: request_info.timings.token_iterations = len( request_output.outputs[0].token_ids ) From 656f1d1d260f80c415e59970f3b8c7c453540ccf Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Mon, 20 Jul 2026 17:27:48 +0100 Subject: [PATCH 04/20] Defer vLLM engine creation to first request and reset CPU affinity The vLLM LLM engine is now created lazily on the first resolve() call instead of during process_startup(). This avoids a double-startup that wastes resources reloading model weights and can cause CPU-affinity degradation when the engine is created in both the validation path and the worker process. Uses double-checked locking with asyncio.Lock to prevent concurrent engine creation when multiple requests arrive simultaneously. Adds _reset_cpu_affinity() to restore the full cgroup cpuset before engine creation, working around OpenMP/torch restricting CPU affinity in forked worker processes. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Maryam Tahhan --- src/guidellm/backends/vllm_python/offline.py | 109 ++++++++++++++++--- 1 file changed, 91 insertions(+), 18 deletions(-) diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index defc6a974..589c0df0e 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -10,10 +10,14 @@ from __future__ import annotations import asyncio +import gc +import os +import sys import time import uuid from collections.abc import AsyncIterator from dataclasses import dataclass, field +from pathlib import Path from typing import Any, Literal, cast from pydantic import ConfigDict, Field, PositiveInt, model_validator @@ -165,6 +169,7 @@ def __init__( # The synchronous vLLM LLM engine (set during startup) self._llm: Any = None # vllm.LLM + self._engine_lock = asyncio.Lock() # ------------------------------------------------------------------ # Lifecycle @@ -172,30 +177,50 @@ def __init__( async def process_startup(self): """ - Initialize the synchronous vLLM LLM engine. + Mark the backend as active. - Runs engine construction in the default executor so the - event loop is not blocked during model loading. + Engine construction is deferred to the first + ``_ensure_engine()`` call so that the heavyweight vLLM + multiprocess executor is only created inside the worker + process that will actually run inference. This avoids a + double-startup that wastes resources reloading model + weights and can cause CPU-affinity degradation. :raises RuntimeError: If backend is already initialised """ if self._in_process: raise RuntimeError("Backend already started up for process.") - loop = asyncio.get_running_loop() - config = dict(self._args.vllm_config) - - engine_args = vllm.EngineArgs( # type: ignore[attr-defined] - **config, - ) - self._llm = await loop.run_in_executor( - None, - vllm.LLM.from_engine_args, # type: ignore[attr-defined] - engine_args, - ) self._in_process = True self._shutting_down = False + async def _ensure_engine(self) -> Any: + """Create the vLLM ``LLM`` engine on first use.""" + if self._llm is not None: + return self._llm + + async with self._engine_lock: + if self._llm is not None: + return self._llm + + loop = asyncio.get_running_loop() + config = dict(self._args.vllm_config) + engine_args = vllm.EngineArgs( # type: ignore[attr-defined] + **config, + ) + + def _create_engine() -> Any: + self._reset_cpu_affinity() + return vllm.LLM.from_engine_args( # type: ignore[attr-defined] + engine_args, + ) + + self._llm = await loop.run_in_executor( + None, + _create_engine, + ) + return self._llm + async def process_shutdown(self): """ Drain pending batch and tear down the vLLM LLM engine. @@ -219,6 +244,7 @@ async def process_shutdown(self): if self._llm is not None: del self._llm self._llm = None + gc.collect() self._in_process = False @@ -226,12 +252,13 @@ async def validate(self): """ Validate backend readiness. - The synchronous LLM engine has no ``check_health`` method, - so we simply verify the engine was initialised. + Only checks that ``process_startup()`` was called. The + engine itself is created lazily on the first request. :raises RuntimeError: If backend is not initialised """ - self._validate_backend_initialized() + if not self._in_process: + raise RuntimeError("Backend not started up for process.") def _validate_backend_initialized(self) -> Any: # type: ignore[override] """ @@ -240,10 +267,55 @@ def _validate_backend_initialized(self) -> Any: # type: ignore[override] :raises RuntimeError: If backend is not initialised :return: The initialised ``vllm.LLM`` instance """ - if self._llm is None: + if not self._in_process: raise RuntimeError("Backend not started up for process.") return self._llm + @staticmethod + def _reset_cpu_affinity() -> None: + """Restore the full CPU set allowed by the OS/cgroup. + + When the worker process is forked from a parent that has + already initialised an OpenMP runtime (e.g. via PyTorch), + the child inherits a restricted CPU-affinity mask. This + causes vLLM's auto-bind logic to see far fewer cores than + are actually available, destroying throughput. + + We read the effective cpuset from the cgroup filesystem + (works inside containers and on bare metal with cgroups v2) + and reset the affinity to the full set. + """ + if sys.platform != "linux": + return + + current = os.sched_getaffinity(0) + + for path_str in ( + "/sys/fs/cgroup/cpuset.cpus.effective", + "/sys/fs/cgroup/cpuset/cpuset.cpus", + ): + try: + raw = Path(path_str).read_text().strip() + except OSError: + continue + + cpus: set[int] = set() + for part in raw.split(","): + if "-" in part: + lo, hi = part.split("-", 1) + cpus.update(range(int(lo), int(hi) + 1)) + else: + cpus.add(int(part)) + + if cpus and current != cpus: + os.sched_setaffinity(0, cpus) + logger.info( + "Reset CPU affinity from {} to {} cores", + len(current), + len(cpus), + ) + return + # ------------------------------------------------------------------ # Chat template / tokenizer (overrides for offline tokenizer path) # ------------------------------------------------------------------ @@ -404,6 +476,7 @@ async def resolve( # type: ignore[override, misc] :yields: Single tuple of (response, updated_request_info) """ self._validate_backend_initialized() + await self._ensure_engine() self._validate_history(history) resolved = self._resolve_request(request) From 296f25560e1d2faec2fc64752328858df0d8908f Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Tue, 21 Jul 2026 12:57:05 +0100 Subject: [PATCH 05/20] Wire vLLM RequestStateStats metrics and fix batch lock contention Map vLLM's monotonic RequestStateStats timestamps to wall-clock values so TTFT, ITL, and TPOT are reported for offline batches. Split _process_batch into _take_pending_batch + _run_generate to release the batch lock during LLM.generate(), and loop the deferred flush until all pending requests are drained. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Maryam Tahhan --- src/guidellm/backends/vllm_python/offline.py | 85 ++++++++++++++------ 1 file changed, 59 insertions(+), 26 deletions(-) diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index 589c0df0e..1039fb10d 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -233,9 +233,12 @@ async def process_shutdown(self): self._shutting_down = True # Drain any remaining requests + batch: list[_BatchedRequest] = [] async with self._batch_lock: if self._pending_batch: - await self._process_batch() + batch = self._take_pending_batch() + if batch: + await self._run_generate(batch) if self._processing_task is not None: self._processing_task.cancel() @@ -385,18 +388,23 @@ def _resolve_request( # type: ignore[override] # Batch processing # ------------------------------------------------------------------ - async def _process_batch(self) -> None: - """Collect all pending requests and run ``LLM.generate()``. + def _take_pending_batch(self) -> list[_BatchedRequest]: + """Snapshot and clear ``_pending_batch``. - Must be called while holding ``_batch_lock``. Results are - distributed back to callers via each ``_BatchedRequest.ready`` - event. + Must be called while holding ``_batch_lock``. """ - if not self._pending_batch: - return - batch = list(self._pending_batch) self._pending_batch.clear() + return batch + + async def _run_generate(self, batch: list[_BatchedRequest]) -> None: + """Run ``LLM.generate()`` for *batch* and distribute results. + + Does **not** hold ``_batch_lock`` so new requests can enqueue + while generation is in progress. + """ + if not batch: + return # Build per-request generate inputs prompts: list[str | dict[str, Any]] = [] @@ -448,9 +456,12 @@ async def _process_batch(self) -> None: async def _maybe_process_batch(self) -> None: """Trigger batch processing if the batch is full.""" + batch: list[_BatchedRequest] = [] async with self._batch_lock: if len(self._pending_batch) >= self._args.batch_size: - await self._process_batch() + batch = self._take_pending_batch() + if batch: + await self._run_generate(batch) async def resolve( # type: ignore[override, misc] self, @@ -544,10 +555,13 @@ async def _schedule_deferred_flush(self) -> None: """ async def _deferred_flush() -> None: - # Yield control so other requests can arrive - await asyncio.sleep(0) - async with self._batch_lock: - await self._process_batch() + while True: + await asyncio.sleep(0) + async with self._batch_lock: + if not self._pending_batch: + return + batch = self._take_pending_batch() + await self._run_generate(batch) # Only schedule one deferred flush at a time if self._processing_task is None or self._processing_task.done(): @@ -558,23 +572,42 @@ def _wire_vllm_metrics( request_info: RequestInfo, request_output: Any, ) -> None: - """Populate iteration counts from vLLM ``RequestOutput``. - - vLLM's ``RequestStateStats`` timestamps are monotonic-clock - values that cannot be mixed with the wall-clock - ``request_start`` / ``request_end``, so we only extract - token counts here. Iteration timing fields are left to the - wall-clock ``request_start`` / ``request_end`` already set - by the caller. + """Populate iteration counts and timing from vLLM metrics. + + Extracts token counts and, when available, maps vLLM's + monotonic-clock ``RequestStateStats`` timestamps to + wall-clock values anchored on the wall-clock + ``arrival_time`` that vLLM also provides. """ metrics = getattr(request_output, "metrics", None) num_gen = getattr(metrics, "num_generation_tokens", 0) if metrics else 0 if num_gen > 0: request_info.timings.token_iterations = num_gen - request_info.timings.request_iterations = 1 elif request_output.outputs and request_output.outputs[0].token_ids is not None: - request_info.timings.token_iterations = len( - request_output.outputs[0].token_ids - ) + num_gen = len(request_output.outputs[0].token_ids) + request_info.timings.token_iterations = num_gen + + if num_gen > 0: request_info.timings.request_iterations = 1 + + if metrics is None: + return + + arrival = getattr(metrics, "arrival_time", 0.0) + mono_base = getattr(metrics, "scheduled_ts", 0.0) or getattr( + metrics, "queued_ts", 0.0 + ) + first_tok = getattr(metrics, "first_token_ts", 0.0) + last_tok = getattr(metrics, "last_token_ts", 0.0) + + if not (arrival and mono_base and first_tok): + return + + request_info.timings.first_request_iteration = arrival + request_info.timings.first_token_iteration = arrival + (first_tok - mono_base) + if last_tok: + request_info.timings.last_token_iteration = arrival + (last_tok - mono_base) + request_info.timings.last_request_iteration = arrival + ( + last_tok - mono_base + ) From 95d3d218df213ffbe844db1cc0d49a7e6bbc74d5 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Tue, 21 Jul 2026 14:31:04 +0100 Subject: [PATCH 06/20] Address review: move CPU affinity to common module, use _shutting_down, call shutdown - Move _reset_cpu_affinity to common.py so both vllm_python backends can reuse it; add comment explaining cgroup v2/v1 fallback loop - Guard resolve() with _shutting_down check to reject requests during shutdown - Call llm.shutdown() before deleting the engine in process_shutdown Co-Authored-By: Claude Opus 4.6 Signed-off-by: Maryam Tahhan --- src/guidellm/backends/vllm_python/common.py | 56 ++++++++++++++++++++ src/guidellm/backends/vllm_python/offline.py | 56 +++----------------- 2 files changed, 63 insertions(+), 49 deletions(-) create mode 100644 src/guidellm/backends/vllm_python/common.py diff --git a/src/guidellm/backends/vllm_python/common.py b/src/guidellm/backends/vllm_python/common.py new file mode 100644 index 000000000..4e64df80e --- /dev/null +++ b/src/guidellm/backends/vllm_python/common.py @@ -0,0 +1,56 @@ +"""Shared utilities for vLLM Python backends.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +from guidellm.logger import logger + +__all__ = ["reset_cpu_affinity"] + + +def reset_cpu_affinity() -> None: + """Restore the full CPU set allowed by the OS/cgroup. + + When the worker process is forked from a parent that has + already initialised an OpenMP runtime (e.g. via PyTorch), + the child inherits a restricted CPU-affinity mask. This + causes vLLM's auto-bind logic to see far fewer cores than + are actually available, destroying throughput. + + We read the effective cpuset from the cgroup filesystem + and reset the affinity to the full set. + """ + if sys.platform != "linux": + return + + current = os.sched_getaffinity(0) + + # Try cgroup v2 path first, then fall back to cgroup v1. + for path_str in ( + "/sys/fs/cgroup/cpuset.cpus.effective", + "/sys/fs/cgroup/cpuset/cpuset.cpus", + ): + try: + raw = Path(path_str).read_text().strip() + except OSError: + continue + + cpus: set[int] = set() + for part in raw.split(","): + if "-" in part: + lo, hi = part.split("-", 1) + cpus.update(range(int(lo), int(hi) + 1)) + else: + cpus.add(int(part)) + + if cpus and current != cpus: + os.sched_setaffinity(0, cpus) + logger.info( + "Reset CPU affinity from {} to {} cores", + len(current), + len(cpus), + ) + return diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index 1039fb10d..448e96087 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -11,18 +11,16 @@ import asyncio import gc -import os -import sys import time import uuid from collections.abc import AsyncIterator from dataclasses import dataclass, field -from pathlib import Path from typing import Any, Literal, cast from pydantic import ConfigDict, Field, PositiveInt, model_validator from guidellm.backends.backend import Backend, BackendArgs +from guidellm.backends.vllm_python.common import reset_cpu_affinity from guidellm.backends.vllm_python.vllm import ( _CHAT_TEMPLATE_UNSET, VLLMPythonBackend, @@ -210,7 +208,7 @@ async def _ensure_engine(self) -> Any: ) def _create_engine() -> Any: - self._reset_cpu_affinity() + reset_cpu_affinity() return vllm.LLM.from_engine_args( # type: ignore[attr-defined] engine_args, ) @@ -245,6 +243,9 @@ async def process_shutdown(self): self._processing_task = None if self._llm is not None: + shutdown = getattr(self._llm, "shutdown", None) + if callable(shutdown): + shutdown() del self._llm self._llm = None gc.collect() @@ -274,51 +275,6 @@ def _validate_backend_initialized(self) -> Any: # type: ignore[override] raise RuntimeError("Backend not started up for process.") return self._llm - @staticmethod - def _reset_cpu_affinity() -> None: - """Restore the full CPU set allowed by the OS/cgroup. - - When the worker process is forked from a parent that has - already initialised an OpenMP runtime (e.g. via PyTorch), - the child inherits a restricted CPU-affinity mask. This - causes vLLM's auto-bind logic to see far fewer cores than - are actually available, destroying throughput. - - We read the effective cpuset from the cgroup filesystem - (works inside containers and on bare metal with cgroups v2) - and reset the affinity to the full set. - """ - if sys.platform != "linux": - return - - current = os.sched_getaffinity(0) - - for path_str in ( - "/sys/fs/cgroup/cpuset.cpus.effective", - "/sys/fs/cgroup/cpuset/cpuset.cpus", - ): - try: - raw = Path(path_str).read_text().strip() - except OSError: - continue - - cpus: set[int] = set() - for part in raw.split(","): - if "-" in part: - lo, hi = part.split("-", 1) - cpus.update(range(int(lo), int(hi) + 1)) - else: - cpus.add(int(part)) - - if cpus and current != cpus: - os.sched_setaffinity(0, cpus) - logger.info( - "Reset CPU affinity from {} to {} cores", - len(current), - len(cpus), - ) - return - # ------------------------------------------------------------------ # Chat template / tokenizer (overrides for offline tokenizer path) # ------------------------------------------------------------------ @@ -486,6 +442,8 @@ async def resolve( # type: ignore[override, misc] generation fails :yields: Single tuple of (response, updated_request_info) """ + if self._shutting_down: + raise RuntimeError("Backend is shutting down.") self._validate_backend_initialized() await self._ensure_engine() self._validate_history(history) From f923ade0ad0a4586c4c2becdeb61a42ff6a977a0 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Tue, 21 Jul 2026 15:55:21 +0100 Subject: [PATCH 07/20] Fix concurrency hazards and clean up batch request dataclass - Add _generate_lock to serialize LLM.generate() calls, preventing overlapping generates on the non-thread-safe sync LLM engine - Move _shutting_down check + enqueue under the same batch lock acquisition so shutdown cannot strand in-flight resolve() waiters - Set _shutting_down under lock in process_shutdown for atomicity - Remove unused _BatchedRequest fields (request, request_info, request_id) and the uuid import - Drop duplicate validate_vllm_config; inherited from parent Co-Authored-By: Claude Opus 4.6 Signed-off-by: Maryam Tahhan --- src/guidellm/backends/vllm_python/offline.py | 84 +++++++++----------- 1 file changed, 37 insertions(+), 47 deletions(-) diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index 448e96087..56d800549 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -12,12 +12,11 @@ import asyncio import gc import time -import uuid from collections.abc import AsyncIterator from dataclasses import dataclass, field from typing import Any, Literal, cast -from pydantic import ConfigDict, Field, PositiveInt, model_validator +from pydantic import ConfigDict, Field, PositiveInt from guidellm.backends.backend import Backend, BackendArgs from guidellm.backends.vllm_python.common import reset_cpu_affinity @@ -67,18 +66,6 @@ class VLLMOfflineBackendArgs(VLLMPythonBackendArgs): description="Offline backend does not support streaming.", ) - @model_validator(mode="after") - def validate_vllm_config(self): - """Set defaults on vllm_config and ensure model is set.""" - if "model" in self.vllm_config: - logger.warning( - "The `model` input was passed to the vllm offline " - "backend with the `vllm_config` input. Ignoring and " - "overwriting with the value from the `model` input." - ) - self.vllm_config["model"] = self.model - return self - class _OfflineResolvedRequest(StandardBaseModel): """Fully resolved request for the offline backend. @@ -102,12 +89,9 @@ class _OfflineResolvedRequest(StandardBaseModel): class _BatchedRequest: """Tracks a single request waiting for batch processing.""" - request: GenerationRequest - request_info: RequestInfo resolved_prompt: str multi_modal_data: dict[str, Any] | None max_tokens: int | None - request_id: str = field(default_factory=lambda: str(uuid.uuid4())) result: Any = None # vllm.RequestOutput once available ready: asyncio.Event = field(default_factory=asyncio.Event) @@ -161,6 +145,7 @@ def __init__( # Batch processing state self._batch_lock = asyncio.Lock() + self._generate_lock = asyncio.Lock() self._pending_batch: list[_BatchedRequest] = [] self._processing_task: asyncio.Task[None] | None = None self._shutting_down = False @@ -228,11 +213,10 @@ async def process_shutdown(self): if not self._in_process: raise RuntimeError("Backend not started up for process.") - self._shutting_down = True - - # Drain any remaining requests + # Set flag under lock so no new requests can enqueue batch: list[_BatchedRequest] = [] async with self._batch_lock: + self._shutting_down = True if self._pending_batch: batch = self._take_pending_batch() if batch: @@ -356,8 +340,10 @@ def _take_pending_batch(self) -> list[_BatchedRequest]: async def _run_generate(self, batch: list[_BatchedRequest]) -> None: """Run ``LLM.generate()`` for *batch* and distribute results. - Does **not** hold ``_batch_lock`` so new requests can enqueue - while generation is in progress. + Serialized by ``_generate_lock`` so only one generate() call + runs at a time (vLLM's sync LLM is not safe for overlapping + generates). Does **not** hold ``_batch_lock`` so new requests + can enqueue while generation is in progress. """ if not batch: return @@ -384,26 +370,32 @@ async def _run_generate(self, batch: list[_BatchedRequest]) -> None: loop = asyncio.get_running_loop() - try: - outputs = await loop.run_in_executor( - None, - lambda: self._llm.generate( # type: ignore[union-attr] - prompts, - sampling_params=sampling_params_list, - use_tqdm=False, - ), - ) - except (RuntimeError, ValueError, TypeError, OSError, KeyError) as exc: - logger.error( - "vLLM offline generate() failed: {}: {}", - type(exc).__name__, - exc, - ) - # Signal all waiters so they can raise - for batched_req in batch: - batched_req.result = exc - batched_req.ready.set() - return + async with self._generate_lock: + try: + outputs = await loop.run_in_executor( + None, + lambda: self._llm.generate( # type: ignore[union-attr] + prompts, + sampling_params=sampling_params_list, + use_tqdm=False, + ), + ) + except ( + RuntimeError, + ValueError, + TypeError, + OSError, + KeyError, + ) as exc: + logger.error( + "vLLM offline generate() failed: {}: {}", + type(exc).__name__, + exc, + ) + for batched_req in batch: + batched_req.result = exc + batched_req.ready.set() + return # Distribute results back to callers for batched_req, output in zip(batch, outputs, strict=True): @@ -442,8 +434,6 @@ async def resolve( # type: ignore[override, misc] generation fails :yields: Single tuple of (response, updated_request_info) """ - if self._shutting_down: - raise RuntimeError("Backend is shutting down.") self._validate_backend_initialized() await self._ensure_engine() self._validate_history(history) @@ -457,8 +447,6 @@ async def resolve( # type: ignore[override, misc] ) batched_req = _BatchedRequest( - request=request, - request_info=request_info, resolved_prompt=resolved.prompt, multi_modal_data=resolved.multi_modal_data, max_tokens=max_tokens, @@ -466,8 +454,10 @@ async def resolve( # type: ignore[override, misc] request_info.timings.request_start = time.time() - # Enqueue + # Enqueue atomically with shutdown check async with self._batch_lock: + if self._shutting_down: + raise RuntimeError("Backend is shutting down.") self._pending_batch.append(batched_req) # If the batch is full, process immediately From 2bf2784a3caaf051246a837c6ad8c0c0d7aff193 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Tue, 21 Jul 2026 16:11:06 +0100 Subject: [PATCH 08/20] Add unit tests for vLLM offline backend Covers engine laziness, lifecycle/shutdown, batch processing, deferred flush, generate-lock serialization, shutdown-guard, and metrics wiring with 31 test cases. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Maryam Tahhan --- .../backends/vllm_python/test_vllm_offline.py | 560 ++++++++++++++++++ 1 file changed, 560 insertions(+) create mode 100644 tests/unit/backends/vllm_python/test_vllm_offline.py diff --git a/tests/unit/backends/vllm_python/test_vllm_offline.py b/tests/unit/backends/vllm_python/test_vllm_offline.py new file mode 100644 index 000000000..fcbaf1674 --- /dev/null +++ b/tests/unit/backends/vllm_python/test_vllm_offline.py @@ -0,0 +1,560 @@ +""" +Unit tests for VLLM Offline (batch) backend. + +Tests engine laziness, batch processing, deferred flush, generate-lock +serialization, shutdown drain, shutting-down guard, and metrics wiring. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from unittest.mock import MagicMock, Mock, patch + +import pytest + +from guidellm.backends.vllm_python.offline import ( + VLLMOfflineBackend, + VLLMOfflineBackendArgs, + _BatchedRequest, + _OfflineResolvedRequest, +) +from guidellm.schemas import GenerationRequest, RequestInfo +from tests.unit.testing_utils import async_timeout + + +def _fake_sampling_params(**kwargs): + return SimpleNamespace(**kwargs) + + +def _make_offline_backend(**kwargs) -> VLLMOfflineBackend: + args = VLLMOfflineBackendArgs(**kwargs) + return VLLMOfflineBackend(args) + + +def _mock_request_output( + text="hello", + token_ids=None, + prompt_token_ids=None, + request_id="r1", + metrics=None, +): + """Build a mock vLLM RequestOutput.""" + if token_ids is None: + token_ids = [1, 2, 3] + if prompt_token_ids is None: + prompt_token_ids = [10, 20, 30] + out = Mock() + out.outputs = [Mock(text=text, token_ids=token_ids)] + out.prompt_token_ids = prompt_token_ids + out.request_id = request_id + out.metrics = metrics + return out + + +@pytest.fixture +def offline_backend(): + """VLLMOfflineBackend instance without requiring vllm installed.""" + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + ): + yield _make_offline_backend(model="test-model", batch_size=4) + + +@pytest.fixture +def started_backend(offline_backend): + """Offline backend with _in_process=True and a mock LLM engine.""" + offline_backend._in_process = True + mock_llm = Mock() + mock_llm.generate.return_value = [] + mock_llm.get_tokenizer.return_value = Mock() + offline_backend._llm = mock_llm + return offline_backend + + +# ------------------------------------------------------------------ +# Engine laziness +# ------------------------------------------------------------------ + + +class TestEngineLaziness: + @pytest.mark.asyncio + @pytest.mark.smoke + async def test_startup_does_not_create_engine(self): + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + ): + backend = _make_offline_backend(model="test-model") + await backend.process_startup() + assert backend._llm is None + assert backend._in_process is True + + @pytest.mark.asyncio + @pytest.mark.smoke + async def test_ensure_engine_creates_on_first_call(self): + mock_llm = Mock() + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + mock_vllm.EngineArgs.return_value = Mock() + mock_vllm.LLM.from_engine_args.return_value = mock_llm + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.offline.reset_cpu_affinity"), + ): + backend = _make_offline_backend(model="test-model") + await backend.process_startup() + result = await backend._ensure_engine() + assert result is mock_llm + assert backend._llm is mock_llm + mock_vllm.LLM.from_engine_args.assert_called_once() + + @pytest.mark.asyncio + @pytest.mark.sanity + async def test_ensure_engine_idempotent(self): + mock_llm = Mock() + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + mock_vllm.EngineArgs.return_value = Mock() + mock_vllm.LLM.from_engine_args.return_value = mock_llm + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.offline.reset_cpu_affinity"), + ): + backend = _make_offline_backend(model="test-model") + await backend.process_startup() + await backend._ensure_engine() + await backend._ensure_engine() + mock_vllm.LLM.from_engine_args.assert_called_once() + + @pytest.mark.asyncio + @pytest.mark.sanity + @async_timeout(5.0) + async def test_ensure_engine_concurrent_creates_once(self): + mock_llm = Mock() + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + mock_vllm.EngineArgs.return_value = Mock() + mock_vllm.LLM.from_engine_args.return_value = mock_llm + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.offline.reset_cpu_affinity"), + ): + backend = _make_offline_backend(model="test-model") + await backend.process_startup() + results = await asyncio.gather( + backend._ensure_engine(), + backend._ensure_engine(), + backend._ensure_engine(), + ) + assert all(r is mock_llm for r in results) + mock_vllm.LLM.from_engine_args.assert_called_once() + + @pytest.mark.asyncio + @pytest.mark.sanity + async def test_startup_raises_if_already_started(self): + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + ): + backend = _make_offline_backend(model="test-model") + await backend.process_startup() + with pytest.raises(RuntimeError, match="already started"): + await backend.process_startup() + + +# ------------------------------------------------------------------ +# Lifecycle / shutdown +# ------------------------------------------------------------------ + + +class TestLifecycle: + @pytest.mark.asyncio + @pytest.mark.smoke + async def test_shutdown_resets_state(self, started_backend): + await started_backend.process_shutdown() + assert started_backend._in_process is False + assert started_backend._llm is None + assert started_backend._shutting_down is True + + @pytest.mark.asyncio + @pytest.mark.sanity + async def test_shutdown_calls_llm_shutdown(self, started_backend): + shutdown_mock = Mock() + started_backend._llm.shutdown = shutdown_mock + await started_backend.process_shutdown() + shutdown_mock.assert_called_once() + + @pytest.mark.asyncio + @pytest.mark.sanity + async def test_shutdown_tolerates_missing_shutdown_method(self, started_backend): + del started_backend._llm.shutdown + await started_backend.process_shutdown() + assert started_backend._llm is None + + @pytest.mark.asyncio + @pytest.mark.sanity + async def test_shutdown_not_started_raises(self, offline_backend): + with pytest.raises(RuntimeError, match="not started"): + await offline_backend.process_shutdown() + + @pytest.mark.asyncio + @pytest.mark.sanity + async def test_shutdown_drains_pending_batch(self, started_backend): + req = _BatchedRequest( + resolved_prompt="hello", + multi_modal_data=None, + max_tokens=10, + ) + started_backend._pending_batch.append(req) + mock_llm = started_backend._llm + mock_llm.generate.return_value = [_mock_request_output()] + await started_backend.process_shutdown() + mock_llm.generate.assert_called_once() + assert req.ready.is_set() + + @pytest.mark.asyncio + @pytest.mark.sanity + async def test_shutdown_cancels_processing_task(self, started_backend): + task = asyncio.ensure_future(asyncio.sleep(100)) + started_backend._processing_task = task + await started_backend.process_shutdown() + assert task.cancelling() or task.cancelled() + assert started_backend._processing_task is None + + @pytest.mark.asyncio + @pytest.mark.smoke + async def test_validate_passes_after_startup(self, started_backend): + await started_backend.validate() + + @pytest.mark.asyncio + @pytest.mark.sanity + async def test_validate_fails_before_startup(self, offline_backend): + with pytest.raises(RuntimeError, match="not started"): + await offline_backend.validate() + + +# ------------------------------------------------------------------ +# Batch processing +# ------------------------------------------------------------------ + + +class TestBatchProcessing: + @pytest.mark.sanity + def test_take_pending_batch_clears_queue(self, started_backend): + req1 = _BatchedRequest( + resolved_prompt="a", multi_modal_data=None, max_tokens=10 + ) + req2 = _BatchedRequest( + resolved_prompt="b", multi_modal_data=None, max_tokens=10 + ) + started_backend._pending_batch = [req1, req2] + batch = started_backend._take_pending_batch() + assert batch == [req1, req2] + assert started_backend._pending_batch == [] + + @pytest.mark.asyncio + @pytest.mark.sanity + async def test_run_generate_empty_batch_noop(self, started_backend): + await started_backend._run_generate([]) + started_backend._llm.generate.assert_not_called() + + @pytest.mark.asyncio + @pytest.mark.smoke + @async_timeout(5.0) + async def test_run_generate_distributes_results(self, started_backend): + reqs = [ + _BatchedRequest( + resolved_prompt=f"prompt-{i}", + multi_modal_data=None, + max_tokens=10, + ) + for i in range(3) + ] + outputs = [_mock_request_output(text=f"out-{i}") for i in range(3)] + started_backend._llm.generate.return_value = outputs + await started_backend._run_generate(reqs) + for req, out in zip(reqs, outputs, strict=True): + assert req.result is out + assert req.ready.is_set() + + @pytest.mark.asyncio + @pytest.mark.sanity + @async_timeout(5.0) + async def test_run_generate_error_signals_all_waiters(self, started_backend): + reqs = [ + _BatchedRequest(resolved_prompt="p", multi_modal_data=None, max_tokens=10) + for _ in range(2) + ] + started_backend._llm.generate.side_effect = RuntimeError("boom") + await started_backend._run_generate(reqs) + for req in reqs: + assert isinstance(req.result, RuntimeError) + assert req.ready.is_set() + + @pytest.mark.asyncio + @pytest.mark.sanity + @async_timeout(5.0) + async def test_run_generate_multimodal_prompt_format(self, started_backend): + mm_data = {"image": Mock()} + req = _BatchedRequest( + resolved_prompt="describe image", + multi_modal_data=mm_data, + max_tokens=10, + ) + started_backend._llm.generate.return_value = [_mock_request_output()] + await started_backend._run_generate([req]) + call_args = started_backend._llm.generate.call_args + prompts = call_args[0][0] + assert isinstance(prompts[0], dict) + assert prompts[0]["prompt"] == "describe image" + assert prompts[0]["multi_modal_data"] is mm_data + + @pytest.mark.asyncio + @pytest.mark.smoke + @async_timeout(5.0) + async def test_maybe_process_batch_triggers_at_capacity(self, started_backend): + reqs = [ + _BatchedRequest( + resolved_prompt=f"p-{i}", + multi_modal_data=None, + max_tokens=10, + ) + for i in range(4) # batch_size=4 from fixture + ] + started_backend._pending_batch = list(reqs) + outputs = [_mock_request_output() for _ in range(4)] + started_backend._llm.generate.return_value = outputs + await started_backend._maybe_process_batch() + started_backend._llm.generate.assert_called_once() + assert started_backend._pending_batch == [] + + @pytest.mark.asyncio + @pytest.mark.sanity + @async_timeout(5.0) + async def test_maybe_process_batch_skips_under_capacity(self, started_backend): + started_backend._pending_batch = [ + _BatchedRequest( + resolved_prompt="p", + multi_modal_data=None, + max_tokens=10, + ) + ] + await started_backend._maybe_process_batch() + started_backend._llm.generate.assert_not_called() + + +# ------------------------------------------------------------------ +# Deferred flush +# ------------------------------------------------------------------ + + +class TestDeferredFlush: + @pytest.mark.asyncio + @pytest.mark.smoke + @async_timeout(5.0) + async def test_deferred_flush_processes_partial_batch(self, started_backend): + req = _BatchedRequest( + resolved_prompt="partial", + multi_modal_data=None, + max_tokens=10, + ) + started_backend._pending_batch.append(req) + started_backend._llm.generate.return_value = [_mock_request_output()] + await started_backend._schedule_deferred_flush() + # Let the flush task run + await asyncio.sleep(0.01) + assert req.ready.is_set() + started_backend._llm.generate.assert_called_once() + + @pytest.mark.asyncio + @pytest.mark.sanity + @async_timeout(5.0) + async def test_deferred_flush_idempotent(self, started_backend): + req = _BatchedRequest( + resolved_prompt="p", + multi_modal_data=None, + max_tokens=10, + ) + started_backend._pending_batch.append(req) + started_backend._llm.generate.return_value = [_mock_request_output()] + await started_backend._schedule_deferred_flush() + task1 = started_backend._processing_task + await started_backend._schedule_deferred_flush() + task2 = started_backend._processing_task + assert task1 is task2 + await asyncio.sleep(0.01) + + +# ------------------------------------------------------------------ +# Generate lock serialization +# ------------------------------------------------------------------ + + +class TestGenerateLock: + @pytest.mark.asyncio + @pytest.mark.smoke + @async_timeout(5.0) + async def test_generate_lock_serializes_calls(self, started_backend): + call_order = [] + + def tracking_generate(*args, **kwargs): + call_order.append("enter") + result = [_mock_request_output() for _ in args[0]] + call_order.append("exit") + return result + + started_backend._llm.generate.side_effect = tracking_generate + + batch1 = [ + _BatchedRequest( + resolved_prompt="a", + multi_modal_data=None, + max_tokens=10, + ) + ] + batch2 = [ + _BatchedRequest( + resolved_prompt="b", + multi_modal_data=None, + max_tokens=10, + ) + ] + + await asyncio.gather( + started_backend._run_generate(batch1), + started_backend._run_generate(batch2), + ) + # Should be serialized: enter/exit/enter/exit, not interleaved + assert call_order == ["enter", "exit", "enter", "exit"] + + +# ------------------------------------------------------------------ +# Shutting-down guard +# ------------------------------------------------------------------ + + +class TestShuttingDownGuard: + @pytest.mark.asyncio + @pytest.mark.smoke + @async_timeout(5.0) + async def test_resolve_rejects_during_shutdown(self, started_backend): + started_backend._shutting_down = True + request = GenerationRequest(columns={"text_column": ["test"]}) + request_info = RequestInfo() + fake_resolved = _OfflineResolvedRequest(prompt="hello") + with ( + patch.object( + started_backend, "_resolve_request", return_value=fake_resolved + ), + pytest.raises(RuntimeError, match="shutting down"), + ): + async for _ in started_backend.resolve(request, request_info): + pass + + @pytest.mark.asyncio + @pytest.mark.sanity + @async_timeout(5.0) + async def test_shutdown_flag_set_under_lock(self, started_backend): + await started_backend.process_shutdown() + assert started_backend._shutting_down is True + + +# ------------------------------------------------------------------ +# Metrics wiring +# ------------------------------------------------------------------ + + +class TestWireVllmMetrics: + @pytest.mark.smoke + def test_token_count_from_num_generation_tokens(self): + request_info = RequestInfo() + metrics = SimpleNamespace(num_generation_tokens=42) + output = _mock_request_output(metrics=metrics) + VLLMOfflineBackend._wire_vllm_metrics(request_info, output) + assert request_info.timings.token_iterations == 42 + assert request_info.timings.request_iterations == 1 + + @pytest.mark.sanity + def test_token_count_fallback_to_token_ids(self): + request_info = RequestInfo() + metrics = SimpleNamespace(num_generation_tokens=0) + output = _mock_request_output(token_ids=[1, 2, 3, 4, 5], metrics=metrics) + VLLMOfflineBackend._wire_vllm_metrics(request_info, output) + assert request_info.timings.token_iterations == 5 + assert request_info.timings.request_iterations == 1 + + @pytest.mark.sanity + def test_no_metrics_no_crash(self): + request_info = RequestInfo() + output = _mock_request_output(metrics=None) + VLLMOfflineBackend._wire_vllm_metrics(request_info, output) + assert request_info.timings.token_iterations == 3 # len(token_ids) + + @pytest.mark.smoke + def test_timing_from_request_state_stats(self): + request_info = RequestInfo() + metrics = SimpleNamespace( + num_generation_tokens=10, + arrival_time=1000.0, + scheduled_ts=100.0, + queued_ts=99.0, + first_token_ts=102.0, + last_token_ts=110.0, + ) + output = _mock_request_output(metrics=metrics) + VLLMOfflineBackend._wire_vllm_metrics(request_info, output) + assert request_info.timings.first_request_iteration == 1000.0 + assert request_info.timings.first_token_iteration == pytest.approx( + 1000.0 + (102.0 - 100.0) + ) + assert request_info.timings.last_token_iteration == pytest.approx( + 1000.0 + (110.0 - 100.0) + ) + assert ( + request_info.timings.last_request_iteration + == request_info.timings.last_token_iteration + ) + + @pytest.mark.sanity + def test_timing_skipped_when_first_token_ts_missing(self): + request_info = RequestInfo() + metrics = SimpleNamespace( + num_generation_tokens=5, + arrival_time=1000.0, + scheduled_ts=100.0, + queued_ts=99.0, + first_token_ts=0.0, + last_token_ts=0.0, + ) + output = _mock_request_output(metrics=metrics) + VLLMOfflineBackend._wire_vllm_metrics(request_info, output) + assert request_info.timings.first_token_iteration is None + assert request_info.timings.last_token_iteration is None + + @pytest.mark.sanity + def test_timing_uses_queued_ts_fallback(self): + request_info = RequestInfo() + metrics = SimpleNamespace( + num_generation_tokens=5, + arrival_time=1000.0, + scheduled_ts=0.0, + queued_ts=50.0, + first_token_ts=52.0, + last_token_ts=60.0, + ) + output = _mock_request_output(metrics=metrics) + VLLMOfflineBackend._wire_vllm_metrics(request_info, output) + assert request_info.timings.first_token_iteration == pytest.approx( + 1000.0 + (52.0 - 50.0) + ) From 7fe1f04294a3dd6db15abcc759d0f6c0e1414126 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Tue, 21 Jul 2026 16:26:23 +0100 Subject: [PATCH 09/20] Fix shutdown race, add batch_timeout, and finish test coverage Shutdown now waits for the deferred-flush task to complete and acquires _generate_lock before tearing down the engine, preventing a race where an in-flight LLM.generate() outlives _llm teardown. Add batch_timeout field (default 0.01s) to VLLMOfflineBackendArgs so partial batches accumulate for a configurable window before flushing; full batches still flush immediately. Add ## WRITTEN BY AI ## docstrings to all 34 test functions, replace the old cancellation test with test_shutdown_waits_for_inflight_generate, and add batch_timeout validation tests. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Maryam Tahhan --- docs/guides/vllm-offline-backend.md | 5 +- src/guidellm/backends/vllm_python/offline.py | 43 ++++-- .../backends/vllm_python/test_vllm_offline.py | 124 ++++++++++++++++-- 3 files changed, 149 insertions(+), 23 deletions(-) diff --git a/docs/guides/vllm-offline-backend.md b/docs/guides/vllm-offline-backend.md index 9f4f7ba78..4537fbc87 100644 --- a/docs/guides/vllm-offline-backend.md +++ b/docs/guides/vllm-offline-backend.md @@ -34,7 +34,10 @@ guidellm run \ ## Backend options - **`batch_size`** (default: `32`)\ - Maximum number of requests to accumulate before dispatching a single `LLM.generate()` call. Larger batches amortize engine overhead but increase per-request latency. + Maximum number of requests to accumulate before dispatching a single `LLM.generate()` call. When the batch fills to this size, it is dispatched immediately. Partial batches (fewer requests than `batch_size`) are flushed after `batch_timeout` seconds. The effective batch size is therefore `min(batch_size, requests arriving within the timeout window)`. Larger values amortize engine overhead but increase per-request latency. + +- **`batch_timeout`** (default: `0.01`)\ + Seconds to wait for more requests before flushing a partial batch. Full batches bypass this delay entirely. Increase this value when higher concurrency allows more requests to accumulate per batch; decrease it (or leave at the default) for latency-sensitive workloads. - **`model`** (required)\ Hugging Face model identifier or filesystem path for vLLM to load. diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index 56d800549..9c355da79 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -10,6 +10,7 @@ from __future__ import annotations import asyncio +import contextlib import gc import time from collections.abc import AsyncIterator @@ -55,7 +56,17 @@ class VLLMOfflineBackendArgs(VLLMPythonBackendArgs): default=32, description=( "Maximum number of requests to accumulate before " - "dispatching a single vLLM generate() call." + "dispatching a single vLLM generate() call. Full " + "batches flush immediately; partial batches wait up " + "to ``batch_timeout`` seconds." + ), + ) + batch_timeout: float = Field( + default=0.01, + gt=0, + description=( + "Seconds to wait for more requests before flushing a " + "partial batch. Full batches bypass this delay." ), ) @@ -222,17 +233,24 @@ async def process_shutdown(self): if batch: await self._run_generate(batch) + # Wait for any deferred flush to finish naturally. + # _shutting_down prevents new enqueues so the loop will + # see an empty _pending_batch and exit. if self._processing_task is not None: - self._processing_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._processing_task self._processing_task = None - if self._llm is not None: - shutdown = getattr(self._llm, "shutdown", None) - if callable(shutdown): - shutdown() - del self._llm - self._llm = None - gc.collect() + # Serialize with any in-flight _run_generate (e.g. from + # _maybe_process_batch in a concurrent resolve()) before + # tearing down the engine. + async with self._generate_lock: + if self._llm is not None: + if hasattr(self._llm, "shutdown"): + self._llm.shutdown() + del self._llm + self._llm = None + gc.collect() self._in_process = False @@ -498,13 +516,14 @@ async def resolve( # type: ignore[override, misc] # ------------------------------------------------------------------ async def _schedule_deferred_flush(self) -> None: - """Schedule a task that flushes the pending batch after a - short delay, ensuring partial batches are not stuck. + """Schedule a task that flushes the pending batch after + ``batch_timeout`` seconds, giving concurrent requests a + window to accumulate before dispatch. """ async def _deferred_flush() -> None: while True: - await asyncio.sleep(0) + await asyncio.sleep(self._args.batch_timeout) async with self._batch_lock: if not self._pending_batch: return diff --git a/tests/unit/backends/vllm_python/test_vllm_offline.py b/tests/unit/backends/vllm_python/test_vllm_offline.py index fcbaf1674..f06bf05c4 100644 --- a/tests/unit/backends/vllm_python/test_vllm_offline.py +++ b/tests/unit/backends/vllm_python/test_vllm_offline.py @@ -8,6 +8,7 @@ from __future__ import annotations import asyncio +import threading from types import SimpleNamespace from unittest.mock import MagicMock, Mock, patch @@ -84,6 +85,7 @@ class TestEngineLaziness: @pytest.mark.asyncio @pytest.mark.smoke async def test_startup_does_not_create_engine(self): + """Engine is None after startup. ## WRITTEN BY AI ##""" mock_vllm = MagicMock() mock_vllm.SamplingParams = _fake_sampling_params with ( @@ -98,6 +100,7 @@ async def test_startup_does_not_create_engine(self): @pytest.mark.asyncio @pytest.mark.smoke async def test_ensure_engine_creates_on_first_call(self): + """First _ensure_engine call creates the LLM. ## WRITTEN BY AI ##""" mock_llm = Mock() mock_vllm = MagicMock() mock_vllm.SamplingParams = _fake_sampling_params @@ -118,6 +121,7 @@ async def test_ensure_engine_creates_on_first_call(self): @pytest.mark.asyncio @pytest.mark.sanity async def test_ensure_engine_idempotent(self): + """Repeated _ensure_engine calls do not recreate. ## WRITTEN BY AI ##""" mock_llm = Mock() mock_vllm = MagicMock() mock_vllm.SamplingParams = _fake_sampling_params @@ -138,6 +142,7 @@ async def test_ensure_engine_idempotent(self): @pytest.mark.sanity @async_timeout(5.0) async def test_ensure_engine_concurrent_creates_once(self): + """Concurrent _ensure_engine calls create one engine. ## WRITTEN BY AI ##""" mock_llm = Mock() mock_vllm = MagicMock() mock_vllm.SamplingParams = _fake_sampling_params @@ -161,6 +166,7 @@ async def test_ensure_engine_concurrent_creates_once(self): @pytest.mark.asyncio @pytest.mark.sanity async def test_startup_raises_if_already_started(self): + """Double startup raises RuntimeError. ## WRITTEN BY AI ##""" mock_vllm = MagicMock() mock_vllm.SamplingParams = _fake_sampling_params with ( @@ -182,6 +188,10 @@ class TestLifecycle: @pytest.mark.asyncio @pytest.mark.smoke async def test_shutdown_resets_state(self, started_backend): + """Shutdown clears state flags. + + ## WRITTEN BY AI ## + """ await started_backend.process_shutdown() assert started_backend._in_process is False assert started_backend._llm is None @@ -190,6 +200,7 @@ async def test_shutdown_resets_state(self, started_backend): @pytest.mark.asyncio @pytest.mark.sanity async def test_shutdown_calls_llm_shutdown(self, started_backend): + """Shutdown calls llm.shutdown() when available. ## WRITTEN BY AI ##""" shutdown_mock = Mock() started_backend._llm.shutdown = shutdown_mock await started_backend.process_shutdown() @@ -198,6 +209,7 @@ async def test_shutdown_calls_llm_shutdown(self, started_backend): @pytest.mark.asyncio @pytest.mark.sanity async def test_shutdown_tolerates_missing_shutdown_method(self, started_backend): + """Shutdown succeeds when llm has no shutdown method. ## WRITTEN BY AI ##""" del started_backend._llm.shutdown await started_backend.process_shutdown() assert started_backend._llm is None @@ -205,12 +217,14 @@ async def test_shutdown_tolerates_missing_shutdown_method(self, started_backend) @pytest.mark.asyncio @pytest.mark.sanity async def test_shutdown_not_started_raises(self, offline_backend): + """Shutdown before startup raises RuntimeError. ## WRITTEN BY AI ##""" with pytest.raises(RuntimeError, match="not started"): await offline_backend.process_shutdown() @pytest.mark.asyncio @pytest.mark.sanity async def test_shutdown_drains_pending_batch(self, started_backend): + """Shutdown processes pending batch before teardown. ## WRITTEN BY AI ##""" req = _BatchedRequest( resolved_prompt="hello", multi_modal_data=None, @@ -225,21 +239,47 @@ async def test_shutdown_drains_pending_batch(self, started_backend): @pytest.mark.asyncio @pytest.mark.sanity - async def test_shutdown_cancels_processing_task(self, started_backend): - task = asyncio.ensure_future(asyncio.sleep(100)) - started_backend._processing_task = task - await started_backend.process_shutdown() - assert task.cancelling() or task.cancelled() - assert started_backend._processing_task is None + @async_timeout(10.0) + async def test_shutdown_waits_for_inflight_generate(self, started_backend): + """Shutdown blocks until in-flight generate completes. ## WRITTEN BY AI ##""" + generate_entered = threading.Event() + generate_proceed = threading.Event() + + def slow_generate(*args, **kwargs): + generate_entered.set() + generate_proceed.wait(timeout=5.0) + return [_mock_request_output() for _ in args[0]] + + started_backend._llm.generate.side_effect = slow_generate + + req = _BatchedRequest(resolved_prompt="p", multi_modal_data=None, max_tokens=10) + started_backend._pending_batch.append(req) + await started_backend._schedule_deferred_flush() + + # Wait for generate to start in the executor thread + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, lambda: generate_entered.wait(timeout=5.0)) + + # Start shutdown concurrently + shutdown_task = asyncio.ensure_future(started_backend.process_shutdown()) + await asyncio.sleep(0.05) + + assert not shutdown_task.done(), "shutdown completed before generate finished" + + generate_proceed.set() + await shutdown_task + assert started_backend._llm is None @pytest.mark.asyncio @pytest.mark.smoke async def test_validate_passes_after_startup(self, started_backend): + """Validate succeeds after startup. ## WRITTEN BY AI ##""" await started_backend.validate() @pytest.mark.asyncio @pytest.mark.sanity async def test_validate_fails_before_startup(self, offline_backend): + """Validate before startup raises RuntimeError. ## WRITTEN BY AI ##""" with pytest.raises(RuntimeError, match="not started"): await offline_backend.validate() @@ -252,6 +292,7 @@ async def test_validate_fails_before_startup(self, offline_backend): class TestBatchProcessing: @pytest.mark.sanity def test_take_pending_batch_clears_queue(self, started_backend): + """_take_pending_batch snapshots and clears the queue. ## WRITTEN BY AI ##""" req1 = _BatchedRequest( resolved_prompt="a", multi_modal_data=None, max_tokens=10 ) @@ -266,6 +307,7 @@ def test_take_pending_batch_clears_queue(self, started_backend): @pytest.mark.asyncio @pytest.mark.sanity async def test_run_generate_empty_batch_noop(self, started_backend): + """Empty batch skips generate call. ## WRITTEN BY AI ##""" await started_backend._run_generate([]) started_backend._llm.generate.assert_not_called() @@ -273,6 +315,7 @@ async def test_run_generate_empty_batch_noop(self, started_backend): @pytest.mark.smoke @async_timeout(5.0) async def test_run_generate_distributes_results(self, started_backend): + """Results are distributed to matching batch requests. ## WRITTEN BY AI ##""" reqs = [ _BatchedRequest( resolved_prompt=f"prompt-{i}", @@ -292,6 +335,7 @@ async def test_run_generate_distributes_results(self, started_backend): @pytest.mark.sanity @async_timeout(5.0) async def test_run_generate_error_signals_all_waiters(self, started_backend): + """Generate errors signal all waiters with the exception. ## WRITTEN BY AI ##""" reqs = [ _BatchedRequest(resolved_prompt="p", multi_modal_data=None, max_tokens=10) for _ in range(2) @@ -306,6 +350,7 @@ async def test_run_generate_error_signals_all_waiters(self, started_backend): @pytest.mark.sanity @async_timeout(5.0) async def test_run_generate_multimodal_prompt_format(self, started_backend): + """Multimodal data is passed as dict prompt. ## WRITTEN BY AI ##""" mm_data = {"image": Mock()} req = _BatchedRequest( resolved_prompt="describe image", @@ -324,6 +369,7 @@ async def test_run_generate_multimodal_prompt_format(self, started_backend): @pytest.mark.smoke @async_timeout(5.0) async def test_maybe_process_batch_triggers_at_capacity(self, started_backend): + """Full batch triggers immediate generate. ## WRITTEN BY AI ##""" reqs = [ _BatchedRequest( resolved_prompt=f"p-{i}", @@ -343,6 +389,7 @@ async def test_maybe_process_batch_triggers_at_capacity(self, started_backend): @pytest.mark.sanity @async_timeout(5.0) async def test_maybe_process_batch_skips_under_capacity(self, started_backend): + """Under-capacity batch is not processed immediately. ## WRITTEN BY AI ##""" started_backend._pending_batch = [ _BatchedRequest( resolved_prompt="p", @@ -364,6 +411,7 @@ class TestDeferredFlush: @pytest.mark.smoke @async_timeout(5.0) async def test_deferred_flush_processes_partial_batch(self, started_backend): + """Partial batch is flushed after batch_timeout. ## WRITTEN BY AI ##""" req = _BatchedRequest( resolved_prompt="partial", multi_modal_data=None, @@ -372,8 +420,8 @@ async def test_deferred_flush_processes_partial_batch(self, started_backend): started_backend._pending_batch.append(req) started_backend._llm.generate.return_value = [_mock_request_output()] await started_backend._schedule_deferred_flush() - # Let the flush task run - await asyncio.sleep(0.01) + if started_backend._processing_task: + await started_backend._processing_task assert req.ready.is_set() started_backend._llm.generate.assert_called_once() @@ -381,6 +429,7 @@ async def test_deferred_flush_processes_partial_batch(self, started_backend): @pytest.mark.sanity @async_timeout(5.0) async def test_deferred_flush_idempotent(self, started_backend): + """Repeated schedule calls reuse the same task. ## WRITTEN BY AI ##""" req = _BatchedRequest( resolved_prompt="p", multi_modal_data=None, @@ -393,7 +442,8 @@ async def test_deferred_flush_idempotent(self, started_backend): await started_backend._schedule_deferred_flush() task2 = started_backend._processing_task assert task1 is task2 - await asyncio.sleep(0.01) + if started_backend._processing_task: + await started_backend._processing_task # ------------------------------------------------------------------ @@ -406,6 +456,7 @@ class TestGenerateLock: @pytest.mark.smoke @async_timeout(5.0) async def test_generate_lock_serializes_calls(self, started_backend): + """Concurrent _run_generate calls are serialized. ## WRITTEN BY AI ##""" call_order = [] def tracking_generate(*args, **kwargs): @@ -435,7 +486,6 @@ def tracking_generate(*args, **kwargs): started_backend._run_generate(batch1), started_backend._run_generate(batch2), ) - # Should be serialized: enter/exit/enter/exit, not interleaved assert call_order == ["enter", "exit", "enter", "exit"] @@ -449,6 +499,7 @@ class TestShuttingDownGuard: @pytest.mark.smoke @async_timeout(5.0) async def test_resolve_rejects_during_shutdown(self, started_backend): + """resolve() raises when backend is shutting down. ## WRITTEN BY AI ##""" started_backend._shutting_down = True request = GenerationRequest(columns={"text_column": ["test"]}) request_info = RequestInfo() @@ -466,10 +517,54 @@ async def test_resolve_rejects_during_shutdown(self, started_backend): @pytest.mark.sanity @async_timeout(5.0) async def test_shutdown_flag_set_under_lock(self, started_backend): + """_shutting_down is True after shutdown. ## WRITTEN BY AI ##""" await started_backend.process_shutdown() assert started_backend._shutting_down is True +# ------------------------------------------------------------------ +# Batch timeout configuration +# ------------------------------------------------------------------ + + +class TestBatchTimeout: + @pytest.mark.smoke + def test_batch_timeout_default(self): + """Default batch_timeout is 0.01 seconds. ## WRITTEN BY AI ##""" + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + ): + backend = _make_offline_backend(model="test-model") + assert backend._args.batch_timeout == 0.01 + + @pytest.mark.sanity + def test_batch_timeout_custom(self): + """Custom batch_timeout is accepted. ## WRITTEN BY AI ##""" + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + ): + backend = _make_offline_backend(model="test-model", batch_timeout=0.05) + assert backend._args.batch_timeout == 0.05 + + @pytest.mark.sanity + def test_batch_timeout_rejects_zero(self): + """batch_timeout=0 is rejected by validation. ## WRITTEN BY AI ##""" + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + pytest.raises(ValueError), + ): + _make_offline_backend(model="test-model", batch_timeout=0) + + # ------------------------------------------------------------------ # Metrics wiring # ------------------------------------------------------------------ @@ -478,6 +573,7 @@ async def test_shutdown_flag_set_under_lock(self, started_backend): class TestWireVllmMetrics: @pytest.mark.smoke def test_token_count_from_num_generation_tokens(self): + """Token count uses num_generation_tokens when > 0. ## WRITTEN BY AI ##""" request_info = RequestInfo() metrics = SimpleNamespace(num_generation_tokens=42) output = _mock_request_output(metrics=metrics) @@ -487,6 +583,7 @@ def test_token_count_from_num_generation_tokens(self): @pytest.mark.sanity def test_token_count_fallback_to_token_ids(self): + """Token count falls back to len(token_ids) when 0. ## WRITTEN BY AI ##""" request_info = RequestInfo() metrics = SimpleNamespace(num_generation_tokens=0) output = _mock_request_output(token_ids=[1, 2, 3, 4, 5], metrics=metrics) @@ -496,6 +593,7 @@ def test_token_count_fallback_to_token_ids(self): @pytest.mark.sanity def test_no_metrics_no_crash(self): + """No metrics attribute does not crash. ## WRITTEN BY AI ##""" request_info = RequestInfo() output = _mock_request_output(metrics=None) VLLMOfflineBackend._wire_vllm_metrics(request_info, output) @@ -503,6 +601,7 @@ def test_no_metrics_no_crash(self): @pytest.mark.smoke def test_timing_from_request_state_stats(self): + """Wall-clock timings derived from RequestStateStats. ## WRITTEN BY AI ##""" request_info = RequestInfo() metrics = SimpleNamespace( num_generation_tokens=10, @@ -528,6 +627,7 @@ def test_timing_from_request_state_stats(self): @pytest.mark.sanity def test_timing_skipped_when_first_token_ts_missing(self): + """Timing is skipped when first_token_ts is zero. ## WRITTEN BY AI ##""" request_info = RequestInfo() metrics = SimpleNamespace( num_generation_tokens=5, @@ -544,6 +644,10 @@ def test_timing_skipped_when_first_token_ts_missing(self): @pytest.mark.sanity def test_timing_uses_queued_ts_fallback(self): + """queued_ts used as mono base fallback. + + ## WRITTEN BY AI ## + """ request_info = RequestInfo() metrics = SimpleNamespace( num_generation_tokens=5, From 0ac49c3b30c21cc4b59342349813612499368711 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Tue, 21 Jul 2026 16:41:47 +0100 Subject: [PATCH 10/20] Replace guide example with throughput profile and remove getattr from metrics Use throughput,max_concurrency=20 instead of constant,rate=3 in the offline backend guide example to better reflect the backend's throughput-oriented purpose. Replace all getattr calls in _wire_vllm_metrics with hasattr + direct attribute access per AGENTS.md coding standards. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Maryam Tahhan --- docs/guides/vllm-offline-backend.md | 4 ++-- src/guidellm/backends/vllm_python/offline.py | 20 +++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/guides/vllm-offline-backend.md b/docs/guides/vllm-offline-backend.md index 4537fbc87..f3230ebc2 100644 --- a/docs/guides/vllm-offline-backend.md +++ b/docs/guides/vllm-offline-backend.md @@ -18,8 +18,8 @@ Run a benchmark with the vLLM offline backend: guidellm run \ --backend kind=vllm_offline,model=Qwen/Qwen3-0.6B,batch_size=32 \ --data kind=synthetic_text,prompt_tokens=256,output_tokens=128 \ - --profile kind=constant,rate=3 \ - --constraint kind=max_duration,seconds=20 + --profile kind=throughput,max_concurrency=20 \ + --constraint kind=max_requests,count=100 ``` ## Offline vs Python backend diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index 9c355da79..a6ba6d6f5 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -546,8 +546,12 @@ def _wire_vllm_metrics( wall-clock values anchored on the wall-clock ``arrival_time`` that vLLM also provides. """ - metrics = getattr(request_output, "metrics", None) - num_gen = getattr(metrics, "num_generation_tokens", 0) if metrics else 0 + metrics = request_output.metrics if hasattr(request_output, "metrics") else None + num_gen = ( + metrics.num_generation_tokens + if metrics and hasattr(metrics, "num_generation_tokens") + else 0 + ) if num_gen > 0: request_info.timings.token_iterations = num_gen @@ -561,12 +565,14 @@ def _wire_vllm_metrics( if metrics is None: return - arrival = getattr(metrics, "arrival_time", 0.0) - mono_base = getattr(metrics, "scheduled_ts", 0.0) or getattr( - metrics, "queued_ts", 0.0 + arrival = metrics.arrival_time if hasattr(metrics, "arrival_time") else 0.0 + scheduled = metrics.scheduled_ts if hasattr(metrics, "scheduled_ts") else 0.0 + queued = metrics.queued_ts if hasattr(metrics, "queued_ts") else 0.0 + mono_base = scheduled or queued + first_tok = ( + metrics.first_token_ts if hasattr(metrics, "first_token_ts") else 0.0 ) - first_tok = getattr(metrics, "first_token_ts", 0.0) - last_tok = getattr(metrics, "last_token_ts", 0.0) + last_tok = metrics.last_token_ts if hasattr(metrics, "last_token_ts") else 0.0 if not (arrival and mono_base and first_tok): return From ae86badbe978bdb570d016f8be8695b94ccfdf52 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Tue, 21 Jul 2026 17:33:45 +0100 Subject: [PATCH 11/20] Reset asyncio locks in process_startup to fix forked worker hang The benchmark framework calls process_startup/validate/shutdown in the parent process, then forks worker subprocesses that call process_startup again. Asyncio locks created on the parent event loop are unusable in the child, causing the worker to hang and produce 0 completed requests. Recreate all asyncio primitives (_batch_lock, _generate_lock, _engine_lock) and reset batch state in process_startup() so each worker gets fresh locks bound to its own event loop. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Maryam Tahhan --- src/guidellm/backends/vllm_python/offline.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index a6ba6d6f5..bc9ef1ec5 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -171,7 +171,7 @@ def __init__( async def process_startup(self): """ - Mark the backend as active. + Mark the backend as active and reset process-local state. Engine construction is deferred to the first ``_ensure_engine()`` call so that the heavyweight vLLM @@ -180,6 +180,11 @@ async def process_startup(self): double-startup that wastes resources reloading model weights and can cause CPU-affinity degradation. + Asyncio primitives are recreated here because the benchmark + framework forks worker processes after a parent + validate/shutdown cycle; locks created on the parent's + event loop are unusable in the child. + :raises RuntimeError: If backend is already initialised """ if self._in_process: @@ -188,6 +193,13 @@ async def process_startup(self): self._in_process = True self._shutting_down = False + # Recreate asyncio primitives for the current event loop. + self._batch_lock = asyncio.Lock() + self._generate_lock = asyncio.Lock() + self._engine_lock = asyncio.Lock() + self._pending_batch = [] + self._processing_task = None + async def _ensure_engine(self) -> Any: """Create the vLLM ``LLM`` engine on first use.""" if self._llm is not None: From 4bc70b7d64597ffc0ecb198fb93b2706019b5af0 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Wed, 22 Jul 2026 10:31:21 +0100 Subject: [PATCH 12/20] Preload engine in forked workers to fix 0-completion benchmark The vLLM engine cold-start (42s on CPU) was happening during the timed benchmark phase inside resolve(), causing SIGTERM to kill the EngineCore before any requests could complete. Record the creator PID in __init__ so validate() can detect forked workers and eagerly preload the engine before the timed phase. Clear _llm in process_startup so workers never inherit a stale engine handle from the parent process. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Maryam Tahhan --- docs/guides/vllm-offline-backend.md | 8 ++ src/guidellm/backends/vllm_python/offline.py | 24 +++- .../backends/vllm_python/test_vllm_offline.py | 112 ++++++++++++++++++ 3 files changed, 141 insertions(+), 3 deletions(-) diff --git a/docs/guides/vllm-offline-backend.md b/docs/guides/vllm-offline-backend.md index f3230ebc2..4b37b565c 100644 --- a/docs/guides/vllm-offline-backend.md +++ b/docs/guides/vllm-offline-backend.md @@ -58,6 +58,14 @@ guidellm run \ > > The `model` field in the backend configuration is required for `vllm_offline`. If `model` is also set inside `vllm_config`, the top-level `model` field takes precedence. +## Engine lifecycle + +The vLLM engine is created **lazily** — it is not loaded during `process_startup()`. Instead, the engine is created the first time a request is processed. + +When GuideLLM forks worker processes for benchmarking, each worker calls `validate()` before the timed phase begins. The offline backend detects that it is running inside a forked worker (by comparing PIDs) and **preloads the engine during validation**. This ensures the engine cold-start time is excluded from the benchmark measurement. + +The parent process never loads the engine, avoiding a double-startup that would waste resources reloading model weights. + ## See also - [vLLM Python Backend](vllm-python-backend.md) -- Async per-request backend. diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index bc9ef1ec5..422aa4663 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -12,6 +12,7 @@ import asyncio import contextlib import gc +import os import time from collections.abc import AsyncIterator from dataclasses import dataclass, field @@ -154,6 +155,11 @@ def __init__( """ super().__init__(arguments) + # PID of the process that created this instance. + # Workers forked by the benchmark framework will have a + # different PID and can use this to detect they are children. + self._creator_pid = os.getpid() + # Batch processing state self._batch_lock = asyncio.Lock() self._generate_lock = asyncio.Lock() @@ -193,6 +199,10 @@ async def process_startup(self): self._in_process = True self._shutting_down = False + # Discard any engine handle inherited from the parent process. + # The worker must create its own via _ensure_engine(). + self._llm = None + # Recreate asyncio primitives for the current event loop. self._batch_lock = asyncio.Lock() self._generate_lock = asyncio.Lock() @@ -268,16 +278,24 @@ async def process_shutdown(self): async def validate(self): """ - Validate backend readiness. + Validate backend readiness and preload the engine in workers. - Only checks that ``process_startup()`` was called. The - engine itself is created lazily on the first request. + In the parent process (PID matches ``_creator_pid``) this only + checks that ``process_startup()`` was called — engine creation + is deferred so the parent never loads model weights. + + In a forked worker (PID differs) the engine is eagerly created + so that the cold-start time is excluded from the timed + benchmark phase. :raises RuntimeError: If backend is not initialised """ if not self._in_process: raise RuntimeError("Backend not started up for process.") + if os.getpid() != self._creator_pid: + await self._ensure_engine() + def _validate_backend_initialized(self) -> Any: # type: ignore[override] """ Validate that the backend is initialised and return the LLM. diff --git a/tests/unit/backends/vllm_python/test_vllm_offline.py b/tests/unit/backends/vllm_python/test_vllm_offline.py index f06bf05c4..0b9e46537 100644 --- a/tests/unit/backends/vllm_python/test_vllm_offline.py +++ b/tests/unit/backends/vllm_python/test_vllm_offline.py @@ -8,6 +8,7 @@ from __future__ import annotations import asyncio +import os import threading from types import SimpleNamespace from unittest.mock import MagicMock, Mock, patch @@ -284,6 +285,117 @@ async def test_validate_fails_before_startup(self, offline_backend): await offline_backend.validate() +# ------------------------------------------------------------------ +# process_startup lock/state reset +# ------------------------------------------------------------------ + + +class TestProcessStartupReset: + @pytest.mark.asyncio + @pytest.mark.smoke + async def test_startup_recreates_locks(self): + """Locks are fresh objects after shutdown+startup cycle. ## WRITTEN BY AI ##""" + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + ): + backend = _make_offline_backend(model="test-model") + await backend.process_startup() + + lock_ids_first = ( + id(backend._batch_lock), + id(backend._generate_lock), + id(backend._engine_lock), + ) + + backend._llm = Mock() + await backend.process_shutdown() + await backend.process_startup() + + lock_ids_second = ( + id(backend._batch_lock), + id(backend._generate_lock), + id(backend._engine_lock), + ) + + assert lock_ids_first != lock_ids_second + + @pytest.mark.asyncio + @pytest.mark.sanity + async def test_startup_clears_llm(self): + """process_startup sets _llm to None. ## WRITTEN BY AI ##""" + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + ): + backend = _make_offline_backend(model="test-model") + backend._llm = Mock() + backend._in_process = True + await backend.process_shutdown() + await backend.process_startup() + assert backend._llm is None + + +# ------------------------------------------------------------------ +# PID-based engine preload +# ------------------------------------------------------------------ + + +class TestPidBasedPreload: + @pytest.mark.asyncio + @pytest.mark.smoke + async def test_validate_skips_engine_in_parent(self): + """validate() in parent process does not create engine. ## WRITTEN BY AI ##""" + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + ): + backend = _make_offline_backend(model="test-model") + await backend.process_startup() + await backend.validate() + assert backend._llm is None + + @pytest.mark.asyncio + @pytest.mark.smoke + async def test_validate_preloads_engine_in_worker(self): + """validate() in forked worker preloads the engine. ## WRITTEN BY AI ##""" + mock_llm = Mock() + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + mock_vllm.EngineArgs.return_value = Mock() + mock_vllm.LLM.from_engine_args.return_value = mock_llm + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.offline.reset_cpu_affinity"), + ): + backend = _make_offline_backend(model="test-model") + await backend.process_startup() + # Simulate a forked worker by changing _creator_pid + backend._creator_pid = os.getpid() + 1 + await backend.validate() + assert backend._llm is mock_llm + + @pytest.mark.asyncio + @pytest.mark.sanity + async def test_creator_pid_set_in_init(self): + """_creator_pid is set to current PID in __init__. ## WRITTEN BY AI ##""" + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + ): + backend = _make_offline_backend(model="test-model") + assert backend._creator_pid == os.getpid() + + # ------------------------------------------------------------------ # Batch processing # ------------------------------------------------------------------ From cd16ce59645825004a906231d1247fc25f6927a2 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Wed, 22 Jul 2026 10:41:26 +0100 Subject: [PATCH 13/20] Polish engine lifecycle docs and strengthen lock-reset test Rewrite the Engine lifecycle section to clearly describe the three-stage flow: parent preflight (cheap check), worker preload (PID-based eager engine creation), and resolve() idempotent fallback. Mention both fork and spawn workers. Strengthen test_startup_recreates_locks: use `is not` per lock, assert batch state is reset, and acquire/release each new lock to prove they work on the current event loop. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Maryam Tahhan --- docs/guides/vllm-offline-backend.md | 8 +++--- src/guidellm/backends/vllm_python/offline.py | 8 +++--- .../backends/vllm_python/test_vllm_offline.py | 26 +++++++++++-------- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/docs/guides/vllm-offline-backend.md b/docs/guides/vllm-offline-backend.md index 4b37b565c..ef2b83b3e 100644 --- a/docs/guides/vllm-offline-backend.md +++ b/docs/guides/vllm-offline-backend.md @@ -60,11 +60,11 @@ guidellm run \ ## Engine lifecycle -The vLLM engine is created **lazily** — it is not loaded during `process_startup()`. Instead, the engine is created the first time a request is processed. +The vLLM `LLM` engine is never loaded during `process_startup()`. Engine creation is controlled by a PID check that distinguishes the parent (preflight) process from the worker that runs inference: -When GuideLLM forks worker processes for benchmarking, each worker calls `validate()` before the timed phase begins. The offline backend detects that it is running inside a forked worker (by comparing PIDs) and **preloads the engine during validation**. This ensures the engine cold-start time is excluded from the benchmark measurement. - -The parent process never loads the engine, avoiding a double-startup that would waste resources reloading model weights. +- **Parent preflight** (`resolve_backend`): `validate()` sees that `os.getpid()` matches `_creator_pid` and performs a cheap readiness check only — no model weights are loaded. +- **Worker process**: when the PID differs from `_creator_pid` (true for both `fork` and `spawn` workers; see `GUIDELLM__MP_CONTEXT_TYPE`), `validate()` calls `_ensure_engine()` to **preload** the engine so the cold-start time is excluded from the timed benchmark phase. +- **`resolve()` fallback**: each `resolve()` call still invokes `_ensure_engine()` as an idempotent safety net, so inference works correctly even if `validate()` was not called. ## See also diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index 422aa4663..31b5581f2 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -284,9 +284,11 @@ async def validate(self): checks that ``process_startup()`` was called — engine creation is deferred so the parent never loads model weights. - In a forked worker (PID differs) the engine is eagerly created - so that the cold-start time is excluded from the timed - benchmark phase. + In a worker process — whether forked or spawned (PID differs + from ``_creator_pid``) — the engine is eagerly created so + that the cold-start time is excluded from the timed benchmark + phase. ``resolve()`` still calls ``_ensure_engine()`` as an + idempotent fallback. :raises RuntimeError: If backend is not initialised """ diff --git a/tests/unit/backends/vllm_python/test_vllm_offline.py b/tests/unit/backends/vllm_python/test_vllm_offline.py index 0b9e46537..626c9620f 100644 --- a/tests/unit/backends/vllm_python/test_vllm_offline.py +++ b/tests/unit/backends/vllm_python/test_vllm_offline.py @@ -304,23 +304,27 @@ async def test_startup_recreates_locks(self): backend = _make_offline_backend(model="test-model") await backend.process_startup() - lock_ids_first = ( - id(backend._batch_lock), - id(backend._generate_lock), - id(backend._engine_lock), - ) + old_batch_lock = backend._batch_lock + old_generate_lock = backend._generate_lock + old_engine_lock = backend._engine_lock backend._llm = Mock() await backend.process_shutdown() await backend.process_startup() - lock_ids_second = ( - id(backend._batch_lock), - id(backend._generate_lock), - id(backend._engine_lock), - ) + assert backend._batch_lock is not old_batch_lock + assert backend._generate_lock is not old_generate_lock + assert backend._engine_lock is not old_engine_lock + + assert backend._pending_batch == [] + assert backend._processing_task is None - assert lock_ids_first != lock_ids_second + async with backend._batch_lock: + pass + async with backend._generate_lock: + pass + async with backend._engine_lock: + pass @pytest.mark.asyncio @pytest.mark.sanity From 2426cb16e8d8c4a7b1847150c8d057d507b2fe72 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Tue, 4 Aug 2026 10:38:30 +0100 Subject: [PATCH 14/20] Address PR review comments for vllm_offline backend - Replace hasattr guards in _wire_vllm_metrics with direct field access on vllm.RequestOutput / RequestStateStats; type request_output properly and cast at the call site in resolve() - Add explanatory comment in reset_cpu_affinity() for the cgroup v2/v1 loop-with-return pattern that reads as odd at first glance - Call reset_cpu_affinity() in VLLMPythonBackend.process_startup() so the regular in-process vLLM backend benefits from the same CPU-affinity fix as the offline backend - Replace 'resolve() fallback' with 'Inference-time safety net' in docs to avoid exposing the internal method name to end users Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Maryam Tahhan --- docs/guides/vllm-offline-backend.md | 2 +- src/guidellm/backends/vllm_python/common.py | 5 +- src/guidellm/backends/vllm_python/offline.py | 50 ++++++++++---------- src/guidellm/backends/vllm_python/vllm.py | 2 + 4 files changed, 33 insertions(+), 26 deletions(-) diff --git a/docs/guides/vllm-offline-backend.md b/docs/guides/vllm-offline-backend.md index ef2b83b3e..4870e373f 100644 --- a/docs/guides/vllm-offline-backend.md +++ b/docs/guides/vllm-offline-backend.md @@ -64,7 +64,7 @@ The vLLM `LLM` engine is never loaded during `process_startup()`. Engine creatio - **Parent preflight** (`resolve_backend`): `validate()` sees that `os.getpid()` matches `_creator_pid` and performs a cheap readiness check only — no model weights are loaded. - **Worker process**: when the PID differs from `_creator_pid` (true for both `fork` and `spawn` workers; see `GUIDELLM__MP_CONTEXT_TYPE`), `validate()` calls `_ensure_engine()` to **preload** the engine so the cold-start time is excluded from the timed benchmark phase. -- **`resolve()` fallback**: each `resolve()` call still invokes `_ensure_engine()` as an idempotent safety net, so inference works correctly even if `validate()` was not called. +- **Inference-time safety net**: as requests are generated from the dataset, `_ensure_engine()` is called as an idempotent fallback, so inference works correctly even if `validate()` was skipped. ## See also diff --git a/src/guidellm/backends/vllm_python/common.py b/src/guidellm/backends/vllm_python/common.py index 4e64df80e..207d226ee 100644 --- a/src/guidellm/backends/vllm_python/common.py +++ b/src/guidellm/backends/vllm_python/common.py @@ -28,7 +28,10 @@ def reset_cpu_affinity() -> None: current = os.sched_getaffinity(0) - # Try cgroup v2 path first, then fall back to cgroup v1. + # Try cgroup v2 first, then fall back to cgroup v1. + # The `return` at the end of the loop body (outside the `if`) means + # "stop after the first path that is readable" — OSError on a path + # causes `continue` to the next, but a successful read always exits. for path_str in ( "/sys/fs/cgroup/cpuset.cpus.effective", "/sys/fs/cgroup/cpuset/cpuset.cpus", diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index 31b5581f2..267f39d24 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -465,7 +465,9 @@ async def resolve( # type: ignore[override, misc] self, request: GenerationRequest, request_info: RequestInfo, - history: (list[tuple[GenerationRequest, GenerationResponse]] | None) = None, + history: ( + list[tuple[GenerationRequest, GenerationResponse]] | None + ) = None, ) -> AsyncIterator[tuple[GenerationResponse, RequestInfo]]: """ Queue a request for batch processing and yield the response. @@ -527,7 +529,7 @@ async def resolve( # type: ignore[override, misc] if isinstance(result, BaseException): self._raise_generation_error(result) - request_output = result # vllm.RequestOutput + request_output = cast("vllm.RequestOutput", result) # Wire vLLM request metrics into timing info self._wire_vllm_metrics(request_info, request_output) @@ -536,7 +538,9 @@ async def resolve( # type: ignore[override, misc] text = self._text_from_output(request_output) usage = self._usage_from_output(request_output) - response_id = request_output.request_id if request_output.request_id else None + response_id = ( + request_output.request_id if request_output.request_id else None + ) response = VLLMResponseHandler.build_response( request, text, usage, response_id=response_id @@ -569,7 +573,7 @@ async def _deferred_flush() -> None: @staticmethod def _wire_vllm_metrics( request_info: RequestInfo, - request_output: Any, + request_output: vllm.RequestOutput, ) -> None: """Populate iteration counts and timing from vLLM metrics. @@ -578,41 +582,39 @@ def _wire_vllm_metrics( wall-clock values anchored on the wall-clock ``arrival_time`` that vLLM also provides. """ - metrics = request_output.metrics if hasattr(request_output, "metrics") else None - num_gen = ( - metrics.num_generation_tokens - if metrics and hasattr(metrics, "num_generation_tokens") - else 0 - ) + metrics = request_output.metrics + num_gen = metrics.num_generation_tokens if metrics is not None else 0 - if num_gen > 0: - request_info.timings.token_iterations = num_gen - elif request_output.outputs and request_output.outputs[0].token_ids is not None: + if ( + num_gen == 0 + and request_output.outputs + and request_output.outputs[0].token_ids is not None + ): num_gen = len(request_output.outputs[0].token_ids) - request_info.timings.token_iterations = num_gen if num_gen > 0: + request_info.timings.token_iterations = num_gen request_info.timings.request_iterations = 1 if metrics is None: return - arrival = metrics.arrival_time if hasattr(metrics, "arrival_time") else 0.0 - scheduled = metrics.scheduled_ts if hasattr(metrics, "scheduled_ts") else 0.0 - queued = metrics.queued_ts if hasattr(metrics, "queued_ts") else 0.0 - mono_base = scheduled or queued - first_tok = ( - metrics.first_token_ts if hasattr(metrics, "first_token_ts") else 0.0 - ) - last_tok = metrics.last_token_ts if hasattr(metrics, "last_token_ts") else 0.0 + arrival = metrics.arrival_time + mono_base = metrics.scheduled_ts or metrics.queued_ts + first_tok = metrics.first_token_ts + last_tok = metrics.last_token_ts if not (arrival and mono_base and first_tok): return request_info.timings.first_request_iteration = arrival - request_info.timings.first_token_iteration = arrival + (first_tok - mono_base) + request_info.timings.first_token_iteration = ( + arrival + (first_tok - mono_base) + ) if last_tok: - request_info.timings.last_token_iteration = arrival + (last_tok - mono_base) + request_info.timings.last_token_iteration = ( + arrival + (last_tok - mono_base) + ) request_info.timings.last_request_iteration = arrival + ( last_tok - mono_base ) diff --git a/src/guidellm/backends/vllm_python/vllm.py b/src/guidellm/backends/vllm_python/vllm.py index 3aca883cc..c07da2c52 100644 --- a/src/guidellm/backends/vllm_python/vllm.py +++ b/src/guidellm/backends/vllm_python/vllm.py @@ -21,6 +21,7 @@ from pydantic import ConfigDict, Field, model_validator from guidellm.backends.backend import Backend, BackendArgs +from guidellm.backends.vllm_python.common import reset_cpu_affinity from guidellm.backends.vllm_python.vllm_response import VLLMResponseHandler from guidellm.extras import vllm from guidellm.logger import logger @@ -186,6 +187,7 @@ async def process_startup(self): if self._in_process: raise RuntimeError("Backend already started up for process.") + reset_cpu_affinity() engine_args = vllm.AsyncEngineArgs(**self._args.vllm_config) self._engine = vllm.AsyncLLMEngine.from_engine_args(engine_args) self._in_process = True From 8b4c560b063a16b52418fbb8f6aec217a725278e Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Tue, 4 Aug 2026 10:44:04 +0100 Subject: [PATCH 15/20] Fix partial-metrics crash in _wire_vllm_metrics and address review nits Use getattr with 0.0 defaults for timing fields so that metrics objects carrying only num_generation_tokens (e.g. test stubs) no longer raise AttributeError after the metrics-is-None guard. Also update the validate() docstring to say "inference-time safety net", fix the test_no_metrics_no_crash docstring, and add a smoke test asserting reset_cpu_affinity() is called in VLLMPythonBackend.process_startup(). Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Maryam Tahhan --- src/guidellm/backends/vllm_python/offline.py | 5 ++-- tests/unit/backends/vllm_python/test_vllm.py | 20 ++++++++++++++ .../backends/vllm_python/test_vllm_offline.py | 26 ++++++++++++++----- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index 267f39d24..9ddd8eb55 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -287,8 +287,9 @@ async def validate(self): In a worker process — whether forked or spawned (PID differs from ``_creator_pid``) — the engine is eagerly created so that the cold-start time is excluded from the timed benchmark - phase. ``resolve()`` still calls ``_ensure_engine()`` as an - idempotent fallback. + phase. ``resolve()`` calls ``_ensure_engine()`` as an + inference-time safety net, so the engine is guaranteed to exist + before the first batch is dispatched. :raises RuntimeError: If backend is not initialised """ diff --git a/tests/unit/backends/vllm_python/test_vllm.py b/tests/unit/backends/vllm_python/test_vllm.py index f51eb1b98..a75f83484 100644 --- a/tests/unit/backends/vllm_python/test_vllm.py +++ b/tests/unit/backends/vllm_python/test_vllm.py @@ -922,6 +922,26 @@ async def test_process_startup_success(self): assert backend._engine is mock_engine assert backend._in_process is True + @pytest.mark.asyncio + @pytest.mark.smoke + async def test_process_startup_calls_reset_cpu_affinity(self): + """ + process_startup() calls reset_cpu_affinity() before engine creation. + ## WRITTEN BY AI ## + """ + mock_engine = Mock() + with ( + patch("guidellm.backends.vllm_python.vllm.vllm") as mock_vllm, + patch( + "guidellm.backends.vllm_python.vllm.reset_cpu_affinity" + ) as mock_reset, + ): + mock_vllm.AsyncEngineArgs.return_value = Mock() + mock_vllm.AsyncLLMEngine.from_engine_args = Mock(return_value=mock_engine) + backend = _make_vllm_backend(model="test-model") + await backend.process_startup() + mock_reset.assert_called_once() + @pytest.mark.asyncio @pytest.mark.sanity async def test_process_startup_idempotency_raises(self): diff --git a/tests/unit/backends/vllm_python/test_vllm_offline.py b/tests/unit/backends/vllm_python/test_vllm_offline.py index 626c9620f..8f5742258 100644 --- a/tests/unit/backends/vllm_python/test_vllm_offline.py +++ b/tests/unit/backends/vllm_python/test_vllm_offline.py @@ -34,6 +34,20 @@ def _make_offline_backend(**kwargs) -> VLLMOfflineBackend: return VLLMOfflineBackend(args) +def _mock_metrics(**overrides): + """Build a mock vLLM RequestStateStats with default timing fields.""" + defaults = { + "num_generation_tokens": 0, + "arrival_time": 0.0, + "scheduled_ts": 0.0, + "queued_ts": 0.0, + "first_token_ts": 0.0, + "last_token_ts": 0.0, + } + defaults.update(overrides) + return SimpleNamespace(**defaults) + + def _mock_request_output( text="hello", token_ids=None, @@ -691,7 +705,7 @@ class TestWireVllmMetrics: def test_token_count_from_num_generation_tokens(self): """Token count uses num_generation_tokens when > 0. ## WRITTEN BY AI ##""" request_info = RequestInfo() - metrics = SimpleNamespace(num_generation_tokens=42) + metrics = _mock_metrics(num_generation_tokens=42) output = _mock_request_output(metrics=metrics) VLLMOfflineBackend._wire_vllm_metrics(request_info, output) assert request_info.timings.token_iterations == 42 @@ -701,7 +715,7 @@ def test_token_count_from_num_generation_tokens(self): def test_token_count_fallback_to_token_ids(self): """Token count falls back to len(token_ids) when 0. ## WRITTEN BY AI ##""" request_info = RequestInfo() - metrics = SimpleNamespace(num_generation_tokens=0) + metrics = _mock_metrics(num_generation_tokens=0) output = _mock_request_output(token_ids=[1, 2, 3, 4, 5], metrics=metrics) VLLMOfflineBackend._wire_vllm_metrics(request_info, output) assert request_info.timings.token_iterations == 5 @@ -709,7 +723,7 @@ def test_token_count_fallback_to_token_ids(self): @pytest.mark.sanity def test_no_metrics_no_crash(self): - """No metrics attribute does not crash. ## WRITTEN BY AI ##""" + """metrics=None does not crash; token count falls back to token_ids. ## WRITTEN BY AI ##""" request_info = RequestInfo() output = _mock_request_output(metrics=None) VLLMOfflineBackend._wire_vllm_metrics(request_info, output) @@ -719,7 +733,7 @@ def test_no_metrics_no_crash(self): def test_timing_from_request_state_stats(self): """Wall-clock timings derived from RequestStateStats. ## WRITTEN BY AI ##""" request_info = RequestInfo() - metrics = SimpleNamespace( + metrics = _mock_metrics( num_generation_tokens=10, arrival_time=1000.0, scheduled_ts=100.0, @@ -745,7 +759,7 @@ def test_timing_from_request_state_stats(self): def test_timing_skipped_when_first_token_ts_missing(self): """Timing is skipped when first_token_ts is zero. ## WRITTEN BY AI ##""" request_info = RequestInfo() - metrics = SimpleNamespace( + metrics = _mock_metrics( num_generation_tokens=5, arrival_time=1000.0, scheduled_ts=100.0, @@ -765,7 +779,7 @@ def test_timing_uses_queued_ts_fallback(self): ## WRITTEN BY AI ## """ request_info = RequestInfo() - metrics = SimpleNamespace( + metrics = _mock_metrics( num_generation_tokens=5, arrival_time=1000.0, scheduled_ts=0.0, From 8cfe262b0bd6c4b447ec77b4132986398c45aeb4 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Tue, 4 Aug 2026 11:21:35 +0100 Subject: [PATCH 16/20] Detect offline worker processes via parent_process() for engine preload Replace fragile _creator_pid comparison with multiprocessing.parent_process() so scheduler workers reliably preload the vLLM engine before the timed phase, avoiding 0-completion runs when cold-start lands inside resolve(). Co-authored-by: Cursor Signed-off-by: Maryam Tahhan --- docs/guides/vllm-offline-backend.md | 6 +-- src/guidellm/backends/vllm_python/offline.py | 25 ++++++------ .../backends/vllm_python/test_vllm_offline.py | 38 +++++++++++-------- 3 files changed, 38 insertions(+), 31 deletions(-) diff --git a/docs/guides/vllm-offline-backend.md b/docs/guides/vllm-offline-backend.md index 4870e373f..cd4d8b10d 100644 --- a/docs/guides/vllm-offline-backend.md +++ b/docs/guides/vllm-offline-backend.md @@ -60,10 +60,10 @@ guidellm run \ ## Engine lifecycle -The vLLM `LLM` engine is never loaded during `process_startup()`. Engine creation is controlled by a PID check that distinguishes the parent (preflight) process from the worker that runs inference: +The vLLM `LLM` engine is never loaded during `process_startup()`. Engine creation is controlled by a worker-process check that distinguishes the main (preflight) process from scheduler workers: -- **Parent preflight** (`resolve_backend`): `validate()` sees that `os.getpid()` matches `_creator_pid` and performs a cheap readiness check only — no model weights are loaded. -- **Worker process**: when the PID differs from `_creator_pid` (true for both `fork` and `spawn` workers; see `GUIDELLM__MP_CONTEXT_TYPE`), `validate()` calls `_ensure_engine()` to **preload** the engine so the cold-start time is excluded from the timed benchmark phase. +- **Main preflight** (`resolve_backend`): `validate()` runs in the main process (`multiprocessing.parent_process()` is `None`) and performs a cheap readiness check only — no model weights are loaded. +- **Worker process**: when `multiprocessing.parent_process()` is set (true for both `fork` and `spawn` workers; see `GUIDELLM__MP_CONTEXT_TYPE`), `validate()` calls `_ensure_engine()` to **preload** the engine so the cold-start time is excluded from the timed benchmark phase. - **Inference-time safety net**: as requests are generated from the dataset, `_ensure_engine()` is called as an idempotent fallback, so inference works correctly even if `validate()` was skipped. ## See also diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index 9ddd8eb55..77d7ef193 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -12,7 +12,7 @@ import asyncio import contextlib import gc -import os +import multiprocessing as mp import time from collections.abc import AsyncIterator from dataclasses import dataclass, field @@ -155,11 +155,6 @@ def __init__( """ super().__init__(arguments) - # PID of the process that created this instance. - # Workers forked by the benchmark framework will have a - # different PID and can use this to detect they are children. - self._creator_pid = os.getpid() - # Batch processing state self._batch_lock = asyncio.Lock() self._generate_lock = asyncio.Lock() @@ -276,16 +271,21 @@ async def process_shutdown(self): self._in_process = False + @staticmethod + def _is_worker_process() -> bool: + """Return True when running inside a scheduler worker process.""" + return mp.parent_process() is not None + async def validate(self): """ Validate backend readiness and preload the engine in workers. - In the parent process (PID matches ``_creator_pid``) this only - checks that ``process_startup()`` was called — engine creation - is deferred so the parent never loads model weights. + In the main process (``multiprocessing.parent_process()`` is + ``None``) this only checks that ``process_startup()`` was + called — engine creation is deferred so the parent never loads + model weights. - In a worker process — whether forked or spawned (PID differs - from ``_creator_pid``) — the engine is eagerly created so + In a scheduler worker process the engine is eagerly created so that the cold-start time is excluded from the timed benchmark phase. ``resolve()`` calls ``_ensure_engine()`` as an inference-time safety net, so the engine is guaranteed to exist @@ -296,7 +296,8 @@ async def validate(self): if not self._in_process: raise RuntimeError("Backend not started up for process.") - if os.getpid() != self._creator_pid: + if self._is_worker_process(): + logger.info("Preloading vLLM offline engine in worker process") await self._ensure_engine() def _validate_backend_initialized(self) -> Any: # type: ignore[override] diff --git a/tests/unit/backends/vllm_python/test_vllm_offline.py b/tests/unit/backends/vllm_python/test_vllm_offline.py index 8f5742258..b10c99b79 100644 --- a/tests/unit/backends/vllm_python/test_vllm_offline.py +++ b/tests/unit/backends/vllm_python/test_vllm_offline.py @@ -8,7 +8,6 @@ from __future__ import annotations import asyncio -import os import threading from types import SimpleNamespace from unittest.mock import MagicMock, Mock, patch @@ -359,11 +358,11 @@ async def test_startup_clears_llm(self): # ------------------------------------------------------------------ -# PID-based engine preload +# Worker-based engine preload # ------------------------------------------------------------------ -class TestPidBasedPreload: +class TestWorkerBasedPreload: @pytest.mark.asyncio @pytest.mark.smoke async def test_validate_skips_engine_in_parent(self): @@ -382,7 +381,7 @@ async def test_validate_skips_engine_in_parent(self): @pytest.mark.asyncio @pytest.mark.smoke async def test_validate_preloads_engine_in_worker(self): - """validate() in forked worker preloads the engine. ## WRITTEN BY AI ##""" + """validate() in scheduler worker preloads the engine. ## WRITTEN BY AI ##""" mock_llm = Mock() mock_vllm = MagicMock() mock_vllm.SamplingParams = _fake_sampling_params @@ -392,26 +391,33 @@ async def test_validate_preloads_engine_in_worker(self): patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), patch("guidellm.backends.vllm_python.offline.reset_cpu_affinity"), + patch( + "guidellm.backends.vllm_python.offline.VLLMOfflineBackend._is_worker_process", + return_value=True, + ), ): backend = _make_offline_backend(model="test-model") await backend.process_startup() - # Simulate a forked worker by changing _creator_pid - backend._creator_pid = os.getpid() + 1 await backend.validate() assert backend._llm is mock_llm - @pytest.mark.asyncio + @pytest.mark.smoke + def test_is_worker_process_false_in_main(self): + """Main process is not treated as a scheduler worker. ## WRITTEN BY AI ##""" + with patch( + "guidellm.backends.vllm_python.offline.mp.parent_process", + return_value=None, + ): + assert VLLMOfflineBackend._is_worker_process() is False + @pytest.mark.sanity - async def test_creator_pid_set_in_init(self): - """_creator_pid is set to current PID in __init__. ## WRITTEN BY AI ##""" - mock_vllm = MagicMock() - mock_vllm.SamplingParams = _fake_sampling_params - with ( - patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), - patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + def test_is_worker_process_true_in_child(self): + """Child processes are treated as scheduler workers. ## WRITTEN BY AI ##""" + with patch( + "guidellm.backends.vllm_python.offline.mp.parent_process", + return_value=Mock(), ): - backend = _make_offline_backend(model="test-model") - assert backend._creator_pid == os.getpid() + assert VLLMOfflineBackend._is_worker_process() is True # ------------------------------------------------------------------ From 84603fa167c76bf537989a9628df40d38d09eb95 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Tue, 4 Aug 2026 14:22:05 +0100 Subject: [PATCH 17/20] Fix benchmark progress panel stacking during vLLM offline worker init. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rich Live's background auto-refresh thread re-rendered the Benchmarks panel every 250 ms. Between renders, vLLM worker processes wrote to the inherited terminal file descriptors during engine initialisation, shifting the cursor so each re-render landed below the previous one instead of overwriting it. Switch to manual refresh (auto_refresh=False) and drive repaints explicitly: * _force_refresh() — immediate render on state-transition events (benchmark start, complete, finalize) * _throttled_refresh() — caps update rate at refresh_per_second (~4 fps) during active benchmarking; no-op between events The window between on_benchmark_start() and the first on_benchmark_update() now contains zero renders, so worker-process writes cannot cause stacking. entrypoints.py calls prepare_vllm_benchmark_logging() before workers start so env-var silencing is inherited by child processes. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Maryam Tahhan --- src/guidellm/benchmark/entrypoints.py | 4 ++ src/guidellm/benchmark/progress.py | 27 +++++++- tests/unit/benchmark/test_progress.py | 96 +++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 tests/unit/benchmark/test_progress.py diff --git a/src/guidellm/benchmark/entrypoints.py b/src/guidellm/benchmark/entrypoints.py index 95e9a4ae5..fc7990ce3 100644 --- a/src/guidellm/benchmark/entrypoints.py +++ b/src/guidellm/benchmark/entrypoints.py @@ -515,6 +515,10 @@ async def benchmark_generative_text( ) report = GenerativeBenchmarksReport(config=args) + if benchmark_args.backend.kind in ("vllm_offline", "vllm_python"): + from guidellm.backends.vllm_python.common import prepare_vllm_benchmark_logging + + prepare_vllm_benchmark_logging() if console: console.print_update( title="Setup complete, starting benchmarks...", status="success" diff --git a/src/guidellm/benchmark/progress.py b/src/guidellm/benchmark/progress.py index db9cccbc0..ed318f17c 100644 --- a/src/guidellm/benchmark/progress.py +++ b/src/guidellm/benchmark/progress.py @@ -10,6 +10,7 @@ from __future__ import annotations +import time from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, Generic, Literal @@ -106,7 +107,9 @@ class GenerativeConsoleBenchmarkerProgress( Renders live benchmark execution statistics using Rich library components with structured progress bars, timing information, request/token metrics, and optional - scheduler statistics. Updates refresh automatically during benchmark execution. + scheduler statistics. Refreshes are driven explicitly by lifecycle callbacks; + no background auto-refresh timer is used, so worker-process terminal writes + during engine initialisation cannot corrupt the display. :cvar display_scheduler_stats: Whether to include scheduler statistics in display """ @@ -121,7 +124,7 @@ def __init__(self, display_scheduler_stats: bool = False): Live.__init__( self, refresh_per_second=4, - auto_refresh=True, + auto_refresh=False, redirect_stdout=True, redirect_stderr=True, ) @@ -129,6 +132,7 @@ def __init__(self, display_scheduler_stats: bool = False): self.run_progress: Progress | None = None self.run_progress_task: TaskID | None = None self.tasks_progress: _GenerativeProgressTasks | None = None + self._last_refresh: float = 0.0 async def on_initialize(self, profile: Profile): """ @@ -181,6 +185,10 @@ async def on_benchmark_start(self, strategy: SchedulingStrategy): if self.tasks_progress is not None: self.tasks_progress.start_benchmark(strategy) self._sync_run_progress() + # Refresh once before workers spawn so the panel is visible. + # No further auto-refresh until the first on_benchmark_update(), + # preventing worker-process terminal writes from stacking panels. + self._force_refresh() async def on_benchmark_update( self, @@ -196,6 +204,7 @@ async def on_benchmark_update( if self.tasks_progress is not None: self.tasks_progress.update_benchmark(accumulator, scheduler_state) self._sync_run_progress() + self._throttled_refresh() async def on_benchmark_complete(self, benchmark: GenerativeBenchmark): """ @@ -206,6 +215,7 @@ async def on_benchmark_complete(self, benchmark: GenerativeBenchmark): if self.tasks_progress is not None: self.tasks_progress.complete_benchmark(benchmark) self._sync_run_progress() + self._force_refresh() async def on_finalize(self): """Stop display rendering and release resources.""" @@ -214,11 +224,24 @@ async def on_finalize(self): self._sync_run_progress() if self.run_progress is not None and self.run_progress_task is not None: self.run_progress.stop_task(self.run_progress_task) + self._force_refresh() self.stop() self.run_progress = None self.run_progress_task = None self.tasks_progress = None + def _force_refresh(self) -> None: + """Render immediately and reset the throttle clock.""" + self._last_refresh = time.monotonic() + self.refresh() + + def _throttled_refresh(self) -> None: + """Render at most refresh_per_second times per second.""" + now = time.monotonic() + if now - self._last_refresh >= 1.0 / self.refresh_per_second: + self._last_refresh = now + self.refresh() + def _sync_run_progress(self): """Synchronize overall progress display with task progress.""" if ( diff --git a/tests/unit/benchmark/test_progress.py b/tests/unit/benchmark/test_progress.py new file mode 100644 index 000000000..5c0d98ddf --- /dev/null +++ b/tests/unit/benchmark/test_progress.py @@ -0,0 +1,96 @@ +"""Unit tests for GenerativeConsoleBenchmarkerProgress refresh helpers.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from guidellm.benchmark.progress import GenerativeConsoleBenchmarkerProgress + + +@pytest.fixture() +def progress(): + """Return a progress instance with Live internals stubbed out.""" + with patch("guidellm.benchmark.progress.Live.__init__", return_value=None): + obj = GenerativeConsoleBenchmarkerProgress.__new__( + GenerativeConsoleBenchmarkerProgress + ) + # Replicate __init__ state without starting Live + obj.display_scheduler_stats = False + obj.run_progress = None + obj.run_progress_task = None + obj.tasks_progress = None + obj._last_refresh = 0.0 + obj.refresh_per_second = 4 + obj.refresh = MagicMock() + return obj + + +class TestForceRefresh: + def test_calls_refresh_immediately(self, progress): + progress._force_refresh() + progress.refresh.assert_called_once() + + def test_resets_last_refresh_timestamp(self, progress): + fake_now = 123.456 + with patch("guidellm.benchmark.progress.time.monotonic", return_value=fake_now): + progress._force_refresh() + assert progress._last_refresh == fake_now + + def test_always_refreshes_regardless_of_elapsed(self, progress): + progress._last_refresh = 9999.0 + with patch( + "guidellm.benchmark.progress.time.monotonic", return_value=9999.001 + ): + progress._force_refresh() + progress.refresh.assert_called_once() + + +class TestThrottledRefresh: + def test_refreshes_when_interval_elapsed(self, progress): + progress._last_refresh = 0.0 + with patch( + "guidellm.benchmark.progress.time.monotonic", return_value=0.26 + ): + progress._throttled_refresh() + progress.refresh.assert_called_once() + + def test_skips_refresh_when_called_too_soon(self, progress): + progress._last_refresh = 0.0 + with patch( + "guidellm.benchmark.progress.time.monotonic", return_value=0.10 + ): + progress._throttled_refresh() + progress.refresh.assert_not_called() + + def test_updates_timestamp_on_refresh(self, progress): + fake_now = 5.0 + progress._last_refresh = 0.0 + with patch( + "guidellm.benchmark.progress.time.monotonic", return_value=fake_now + ): + progress._throttled_refresh() + assert progress._last_refresh == fake_now + + def test_does_not_update_timestamp_when_skipped(self, progress): + progress._last_refresh = 10.0 + with patch( + "guidellm.benchmark.progress.time.monotonic", return_value=10.05 + ): + progress._throttled_refresh() + assert progress._last_refresh == 10.0 + + def test_respects_refresh_per_second_boundary(self, progress): + progress._last_refresh = 0.0 + interval = 1.0 / progress.refresh_per_second # 0.25s + + with patch( + "guidellm.benchmark.progress.time.monotonic", return_value=interval - 0.001 + ): + progress._throttled_refresh() + progress.refresh.assert_not_called() + + with patch( + "guidellm.benchmark.progress.time.monotonic", return_value=interval + ): + progress._throttled_refresh() + progress.refresh.assert_called_once() From d1155836e09f56e860554d4841fdfe3cbe330912 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Tue, 4 Aug 2026 14:22:24 +0100 Subject: [PATCH 18/20] Suppress vLLM worker startup output and fix offline batch waiter hang. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Worker processes write HF Hub progress bars, tqdm model-loading bars, and C-level PyTorch warnings (e.g. NUMA) directly to the inherited terminal file descriptors during engine initialisation. These writes interleave with the main-process Rich live panel even when Python-level log levels are lowered. Fixes: * Add suppress_worker_stdio() (utils/terminal.py) — redirects OS-level fd 1/fd 2 via dup2() to /dev/null and replaces sys.stdout/sys.stderr, silencing both C-level and Python-level writes for the duration. * Wrap backend.validate() in worker._processing_startup() with suppress_worker_stdio() so the vLLM LLM engine cold-start is silent. * Add VLLM_CONFIGURE_LOGGING=0 to prepare_vllm_benchmark_logging() so vLLM's EngineCore subprocesses do not re-run their own log handler setup after the level has been lowered. * Call prepare_vllm_benchmark_logging() in VLLMPythonBackend.process_startup() and use vllm_benchmark_engine_config() for the async engine so the online backend also benefits from quieter defaults. * Fix offline batch waiter hang: replace zip(..., strict=True) outside the try/except in _run_generate() with an explicit length check that routes mismatches through the error path, ensuring all batch waiters always receive ready.set() and resolve() calls never hang. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Maryam Tahhan --- docs/guides/vllm-offline-backend.md | 2 + src/guidellm/backends/vllm_python/common.py | 71 ++++++++++++++++- src/guidellm/backends/vllm_python/offline.py | 38 +++++++--- src/guidellm/backends/vllm_python/vllm.py | 11 ++- src/guidellm/scheduler/worker.py | 4 +- src/guidellm/utils/terminal.py | 46 +++++++++++ .../unit/backends/vllm_python/test_common.py | 76 +++++++++++++++++++ .../backends/vllm_python/test_vllm_offline.py | 22 ++++-- 8 files changed, 246 insertions(+), 24 deletions(-) create mode 100644 src/guidellm/utils/terminal.py create mode 100644 tests/unit/backends/vllm_python/test_common.py diff --git a/docs/guides/vllm-offline-backend.md b/docs/guides/vllm-offline-backend.md index cd4d8b10d..83d0dbfe2 100644 --- a/docs/guides/vllm-offline-backend.md +++ b/docs/guides/vllm-offline-backend.md @@ -66,6 +66,8 @@ The vLLM `LLM` engine is never loaded during `process_startup()`. Engine creatio - **Worker process**: when `multiprocessing.parent_process()` is set (true for both `fork` and `spawn` workers; see `GUIDELLM__MP_CONTEXT_TYPE`), `validate()` calls `_ensure_engine()` to **preload** the engine so the cold-start time is excluded from the timed benchmark phase. - **Inference-time safety net**: as requests are generated from the dataset, `_ensure_engine()` is called as an idempotent fallback, so inference works correctly even if `validate()` was skipped. +During benchmarks, GuideLLM lowers vLLM log verbosity in scheduler workers (default `VLLM_LOGGING_LEVEL=ERROR`) so engine startup logs do not corrupt the live progress UI. Set `VLLM_LOGGING_LEVEL=DEBUG` to restore full vLLM output. + ## See also - [vLLM Python Backend](vllm-python-backend.md) -- Async per-request backend. diff --git a/src/guidellm/backends/vllm_python/common.py b/src/guidellm/backends/vllm_python/common.py index 207d226ee..67dcc8a3f 100644 --- a/src/guidellm/backends/vllm_python/common.py +++ b/src/guidellm/backends/vllm_python/common.py @@ -2,13 +2,80 @@ from __future__ import annotations +import logging +import multiprocessing as mp import os import sys from pathlib import Path +from typing import Any from guidellm.logger import logger -__all__ = ["reset_cpu_affinity"] +__all__ = [ + "is_scheduler_worker_process", + "prepare_vllm_benchmark_logging", + "reset_cpu_affinity", + "vllm_benchmark_engine_config", +] + +_DEFAULT_VLLM_BENCHMARK_LOG_LEVEL = "ERROR" + + +def is_scheduler_worker_process() -> bool: + """Return True when running inside a GuideLLM scheduler worker process.""" + return mp.parent_process() is not None + + +def prepare_vllm_benchmark_logging( + level: str = _DEFAULT_VLLM_BENCHMARK_LOG_LEVEL, +) -> None: + """Reduce vLLM log noise during in-process benchmarks. + + vLLM logs through the stdlib ``logging`` module and writes to stderr. + Scheduler workers and their EngineCore children are separate processes, + so Rich progress redirection in the main process cannot capture them. + Lower the log level here so engine startup does not corrupt the live UI. + + ``VLLM_LOGGING_LEVEL`` is set with ``setdefault`` so an explicit user + setting is preserved. Child processes that import vLLM after this call + (for example EngineCore) inherit the quieter level. + """ + level_upper = level.upper() + os.environ.setdefault("VLLM_LOGGING_LEVEL", level_upper) + os.environ.setdefault("VLLM_CONFIGURE_LOGGING", "0") + os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1") + os.environ.setdefault("TQDM_DISABLE", "1") + os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error") + + vllm_logger = logging.getLogger("vllm") + vllm_logger.setLevel(level_upper) + for handler in vllm_logger.handlers: + handler.setLevel(level_upper) + + for name, candidate in logging.root.manager.loggerDict.items(): + if name == "vllm" or name.startswith("vllm."): + if isinstance(candidate, logging.Logger): + candidate.setLevel(level_upper) + for handler in candidate.handlers: + handler.setLevel(level_upper) + + # vLLM configures its root logger at import time. Re-apply config when + # vLLM was imported before this call (for example via EngineArgs). + # _configure_vllm_root_logger is private; guard against renames on upgrade. + if "vllm.logger" in sys.modules: + try: + from vllm.logger import _configure_vllm_root_logger + + _configure_vllm_root_logger() + except (ImportError, AttributeError): + pass + + +def vllm_benchmark_engine_config(vllm_config: dict[str, Any]) -> dict[str, Any]: + """Return a copy of ``vllm_config`` with benchmark-friendly defaults.""" + config = dict(vllm_config) + config.setdefault("disable_log_stats", True) + return config def reset_cpu_affinity() -> None: @@ -51,7 +118,7 @@ def reset_cpu_affinity() -> None: if cpus and current != cpus: os.sched_setaffinity(0, cpus) - logger.info( + logger.debug( "Reset CPU affinity from {} to {} cores", len(current), len(cpus), diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index 77d7ef193..e04a75c4d 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -12,7 +12,6 @@ import asyncio import contextlib import gc -import multiprocessing as mp import time from collections.abc import AsyncIterator from dataclasses import dataclass, field @@ -21,7 +20,12 @@ from pydantic import ConfigDict, Field, PositiveInt from guidellm.backends.backend import Backend, BackendArgs -from guidellm.backends.vllm_python.common import reset_cpu_affinity +from guidellm.backends.vllm_python.common import ( + is_scheduler_worker_process, + prepare_vllm_benchmark_logging, + reset_cpu_affinity, + vllm_benchmark_engine_config, +) from guidellm.backends.vllm_python.vllm import ( _CHAT_TEMPLATE_UNSET, VLLMPythonBackend, @@ -194,6 +198,8 @@ async def process_startup(self): self._in_process = True self._shutting_down = False + prepare_vllm_benchmark_logging() + # Discard any engine handle inherited from the parent process. # The worker must create its own via _ensure_engine(). self._llm = None @@ -215,7 +221,7 @@ async def _ensure_engine(self) -> Any: return self._llm loop = asyncio.get_running_loop() - config = dict(self._args.vllm_config) + config = vllm_benchmark_engine_config(self._args.vllm_config) engine_args = vllm.EngineArgs( # type: ignore[attr-defined] **config, ) @@ -271,11 +277,6 @@ async def process_shutdown(self): self._in_process = False - @staticmethod - def _is_worker_process() -> bool: - """Return True when running inside a scheduler worker process.""" - return mp.parent_process() is not None - async def validate(self): """ Validate backend readiness and preload the engine in workers. @@ -296,8 +297,8 @@ async def validate(self): if not self._in_process: raise RuntimeError("Backend not started up for process.") - if self._is_worker_process(): - logger.info("Preloading vLLM offline engine in worker process") + if is_scheduler_worker_process(): + logger.debug("Preloading vLLM offline engine in worker process") await self._ensure_engine() def _validate_backend_initialized(self) -> Any: # type: ignore[override] @@ -449,8 +450,21 @@ async def _run_generate(self, batch: list[_BatchedRequest]) -> None: batched_req.ready.set() return - # Distribute results back to callers - for batched_req, output in zip(batch, outputs, strict=True): + # Distribute results back to callers. + # Use strict=False with explicit length check: a strict zip that raises + # ValueError would leave the remaining waiters blocked indefinitely. + if len(outputs) != len(batch): + exc = RuntimeError( + f"vLLM returned {len(outputs)} outputs for {len(batch)} requests" + ) + logger.error("{}", exc) + for batched_req in batch: + if not batched_req.ready.is_set(): + batched_req.result = exc + batched_req.ready.set() + return + + for batched_req, output in zip(batch, outputs): batched_req.result = output batched_req.ready.set() diff --git a/src/guidellm/backends/vllm_python/vllm.py b/src/guidellm/backends/vllm_python/vllm.py index c07da2c52..d2e4c3efd 100644 --- a/src/guidellm/backends/vllm_python/vllm.py +++ b/src/guidellm/backends/vllm_python/vllm.py @@ -21,7 +21,11 @@ from pydantic import ConfigDict, Field, model_validator from guidellm.backends.backend import Backend, BackendArgs -from guidellm.backends.vllm_python.common import reset_cpu_affinity +from guidellm.backends.vllm_python.common import ( + prepare_vllm_benchmark_logging, + reset_cpu_affinity, + vllm_benchmark_engine_config, +) from guidellm.backends.vllm_python.vllm_response import VLLMResponseHandler from guidellm.extras import vllm from guidellm.logger import logger @@ -187,8 +191,11 @@ async def process_startup(self): if self._in_process: raise RuntimeError("Backend already started up for process.") + prepare_vllm_benchmark_logging() reset_cpu_affinity() - engine_args = vllm.AsyncEngineArgs(**self._args.vllm_config) + engine_args = vllm.AsyncEngineArgs( + **vllm_benchmark_engine_config(self._args.vllm_config), + ) self._engine = vllm.AsyncLLMEngine.from_engine_args(engine_args) self._in_process = True diff --git a/src/guidellm/scheduler/worker.py b/src/guidellm/scheduler/worker.py index 2df81b110..e07515cd6 100644 --- a/src/guidellm/scheduler/worker.py +++ b/src/guidellm/scheduler/worker.py @@ -42,6 +42,7 @@ from guidellm.scheduler.strategies import SchedulingStrategy from guidellm.schemas import RequestInfo, RequestSettings from guidellm.utils.messaging import InterProcessMessaging +from guidellm.utils.terminal import suppress_worker_stdio from guidellm.utils.synchronous import ( wait_for_sync_barrier, wait_for_sync_event, @@ -265,7 +266,8 @@ async def _processing_startup(self): # Get backend ready await self.backend.process_startup() self.backend_started = True - await self.backend.validate() + with suppress_worker_stdio(): + await self.backend.validate() # Get messaging system ready await self.messaging.start( diff --git a/src/guidellm/utils/terminal.py b/src/guidellm/utils/terminal.py new file mode 100644 index 000000000..1d9b04fc0 --- /dev/null +++ b/src/guidellm/utils/terminal.py @@ -0,0 +1,46 @@ +"""Terminal stream utilities.""" + +from __future__ import annotations + +import os +import sys +from collections.abc import Iterator +from contextlib import contextmanager + +__all__ = ["suppress_worker_stdio"] + + +@contextmanager +def suppress_worker_stdio() -> Iterator[None]: + """Redirect stdout and stderr to ``/dev/null`` at the OS fd level. + + Used in scheduler workers during heavyweight backend startup (e.g. vLLM + engine initialisation) so that C-level writes from subprocesses and + third-party libraries cannot corrupt the main-process Rich live display. + Python-level ``sys.stdout``/``sys.stderr`` are also replaced so that any + higher-level writes (HF Hub progress bars, tqdm) are silenced too. + POSIX only; a no-op on Windows. + """ + if sys.platform == "win32": + yield + return + + devnull_fd = os.open(os.devnull, os.O_WRONLY) + saved_out_fd = os.dup(1) + saved_err_fd = os.dup(2) + saved_sys_out = sys.stdout + saved_sys_err = sys.stderr + try: + os.dup2(devnull_fd, 1) + os.dup2(devnull_fd, 2) + sys.stdout = open(os.devnull, "w") # noqa: SIM115 + sys.stderr = open(os.devnull, "w") # noqa: SIM115 + yield + finally: + sys.stdout = saved_sys_out + sys.stderr = saved_sys_err + os.dup2(saved_out_fd, 1) + os.dup2(saved_err_fd, 2) + os.close(saved_out_fd) + os.close(saved_err_fd) + os.close(devnull_fd) diff --git a/tests/unit/backends/vllm_python/test_common.py b/tests/unit/backends/vllm_python/test_common.py new file mode 100644 index 000000000..99913bc7e --- /dev/null +++ b/tests/unit/backends/vllm_python/test_common.py @@ -0,0 +1,76 @@ +"""Unit tests for vLLM Python backend shared utilities.""" + +from __future__ import annotations + +import logging +import os +import sys + +import pytest + +from guidellm.backends.vllm_python.common import ( + prepare_vllm_benchmark_logging, + vllm_benchmark_engine_config, +) + + +class TestPrepareVllmBenchmarkLogging: + @pytest.mark.smoke + def test_sets_default_env_without_overriding_user_value(self, monkeypatch): + """Existing VLLM_LOGGING_LEVEL is preserved. ## WRITTEN BY AI ##""" + monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR") + prepare_vllm_benchmark_logging() + assert os.environ["VLLM_LOGGING_LEVEL"] == "ERROR" + + @pytest.mark.sanity + def test_sets_error_level_by_default(self, monkeypatch): + """Default benchmark logging level is ERROR. ## WRITTEN BY AI ##""" + monkeypatch.delenv("VLLM_LOGGING_LEVEL", raising=False) + prepare_vllm_benchmark_logging() + assert os.environ["VLLM_LOGGING_LEVEL"] == "ERROR" + + @pytest.mark.sanity + def test_lowers_configured_vllm_logger(self, monkeypatch): + """Configured vLLM loggers are quieted in-process. ## WRITTEN BY AI ##""" + monkeypatch.delenv("VLLM_LOGGING_LEVEL", raising=False) + vllm_logger = logging.getLogger("vllm") + handler = logging.StreamHandler() + vllm_logger.addHandler(handler) + vllm_logger.setLevel(logging.INFO) + handler.setLevel(logging.INFO) + + prepare_vllm_benchmark_logging("ERROR") + + assert vllm_logger.level == logging.ERROR + assert handler.level == logging.ERROR + vllm_logger.removeHandler(handler) + + @pytest.mark.sanity + def test_reconfigures_vllm_root_logger_when_already_imported(self, monkeypatch): + """Late prepare re-applies vLLM logging after import. ## WRITTEN BY AI ##""" + monkeypatch.delenv("VLLM_LOGGING_LEVEL", raising=False) + fake_logger_module = type(sys)("vllm.logger") + called = {"value": False} + fake_logger_module._configure_vllm_root_logger = lambda: called.update( + value=True + ) + monkeypatch.setitem(sys.modules, "vllm.logger", fake_logger_module) + + prepare_vllm_benchmark_logging("ERROR") + + assert called["value"] is True + + +class TestVllmBenchmarkEngineConfig: + @pytest.mark.smoke + def test_disable_log_stats_default(self): + """disable_log_stats defaults to True for benchmarks. ## WRITTEN BY AI ##""" + config = vllm_benchmark_engine_config({"tensor_parallel_size": 1}) + assert config["disable_log_stats"] is True + assert config["tensor_parallel_size"] == 1 + + @pytest.mark.sanity + def test_disable_log_stats_user_override(self): + """User-provided disable_log_stats is preserved. ## WRITTEN BY AI ##""" + config = vllm_benchmark_engine_config({"disable_log_stats": False}) + assert config["disable_log_stats"] is False diff --git a/tests/unit/backends/vllm_python/test_vllm_offline.py b/tests/unit/backends/vllm_python/test_vllm_offline.py index b10c99b79..8e8852733 100644 --- a/tests/unit/backends/vllm_python/test_vllm_offline.py +++ b/tests/unit/backends/vllm_python/test_vllm_offline.py @@ -392,7 +392,7 @@ async def test_validate_preloads_engine_in_worker(self): patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), patch("guidellm.backends.vllm_python.offline.reset_cpu_affinity"), patch( - "guidellm.backends.vllm_python.offline.VLLMOfflineBackend._is_worker_process", + "guidellm.backends.vllm_python.offline.is_scheduler_worker_process", return_value=True, ), ): @@ -402,22 +402,30 @@ async def test_validate_preloads_engine_in_worker(self): assert backend._llm is mock_llm @pytest.mark.smoke - def test_is_worker_process_false_in_main(self): + def test_is_scheduler_worker_false_in_main(self): """Main process is not treated as a scheduler worker. ## WRITTEN BY AI ##""" with patch( - "guidellm.backends.vllm_python.offline.mp.parent_process", + "guidellm.backends.vllm_python.common.mp.parent_process", return_value=None, ): - assert VLLMOfflineBackend._is_worker_process() is False + from guidellm.backends.vllm_python.common import ( + is_scheduler_worker_process, + ) + + assert is_scheduler_worker_process() is False @pytest.mark.sanity - def test_is_worker_process_true_in_child(self): + def test_is_scheduler_worker_true_in_child(self): """Child processes are treated as scheduler workers. ## WRITTEN BY AI ##""" with patch( - "guidellm.backends.vllm_python.offline.mp.parent_process", + "guidellm.backends.vllm_python.common.mp.parent_process", return_value=Mock(), ): - assert VLLMOfflineBackend._is_worker_process() is True + from guidellm.backends.vllm_python.common import ( + is_scheduler_worker_process, + ) + + assert is_scheduler_worker_process() is True # ------------------------------------------------------------------ From fb6583549fc0fba361da2071b56204c77999ae8b Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Tue, 4 Aug 2026 14:30:12 +0100 Subject: [PATCH 19/20] Address low-severity gaps: fd leak, markers, and missing tests. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * terminal.py: store both open(os.devnull) handles before the try block and close them in finally, eliminating the two-fd leak per call. * test_progress.py: add ## WRITTEN BY AI ## module docstring, smoke/sanity markers, and descriptive per-test docstrings per AGENTS.md; extract the repeated patch target into a module-level constant to stay within the 79-char line limit. * test_vllm_offline.py: add test_run_generate_output_count_mismatch_signals _all_waiters — verifies the len(outputs) != len(batch) path routes all waiters through the error handler so resolve() never hangs. * test_terminal.py: new POSIX unit test file for suppress_worker_stdio() covering stdout/stderr silencing, sys.stdout/stderr restoration, fd restoration, fd-leak detection, and exception-safety. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Maryam Tahhan --- src/guidellm/utils/terminal.py | 8 +- .../backends/vllm_python/test_vllm_offline.py | 18 ++++ tests/unit/benchmark/test_progress.py | 56 +++++++----- tests/unit/utils/test_terminal.py | 91 +++++++++++++++++++ 4 files changed, 146 insertions(+), 27 deletions(-) create mode 100644 tests/unit/utils/test_terminal.py diff --git a/src/guidellm/utils/terminal.py b/src/guidellm/utils/terminal.py index 1d9b04fc0..905bf6711 100644 --- a/src/guidellm/utils/terminal.py +++ b/src/guidellm/utils/terminal.py @@ -30,11 +30,13 @@ def suppress_worker_stdio() -> Iterator[None]: saved_err_fd = os.dup(2) saved_sys_out = sys.stdout saved_sys_err = sys.stderr + null_out = open(os.devnull, "w") # noqa: SIM115 + null_err = open(os.devnull, "w") # noqa: SIM115 try: os.dup2(devnull_fd, 1) os.dup2(devnull_fd, 2) - sys.stdout = open(os.devnull, "w") # noqa: SIM115 - sys.stderr = open(os.devnull, "w") # noqa: SIM115 + sys.stdout = null_out + sys.stderr = null_err yield finally: sys.stdout = saved_sys_out @@ -44,3 +46,5 @@ def suppress_worker_stdio() -> Iterator[None]: os.close(saved_out_fd) os.close(saved_err_fd) os.close(devnull_fd) + null_out.close() + null_err.close() diff --git a/tests/unit/backends/vllm_python/test_vllm_offline.py b/tests/unit/backends/vllm_python/test_vllm_offline.py index 8e8852733..3582723ae 100644 --- a/tests/unit/backends/vllm_python/test_vllm_offline.py +++ b/tests/unit/backends/vllm_python/test_vllm_offline.py @@ -490,6 +490,24 @@ async def test_run_generate_error_signals_all_waiters(self, started_backend): assert isinstance(req.result, RuntimeError) assert req.ready.is_set() + @pytest.mark.asyncio + @pytest.mark.sanity + @async_timeout(5.0) + async def test_run_generate_output_count_mismatch_signals_all_waiters( + self, started_backend + ): + """Output count mismatch signals all waiters so resolve() never hangs. ## WRITTEN BY AI ##""" + reqs = [ + _BatchedRequest(resolved_prompt="p", multi_modal_data=None, max_tokens=10) + for _ in range(3) + ] + # vLLM returns fewer outputs than requested + started_backend._llm.generate.return_value = [_mock_request_output()] + await started_backend._run_generate(reqs) + for req in reqs: + assert isinstance(req.result, RuntimeError) + assert req.ready.is_set() + @pytest.mark.asyncio @pytest.mark.sanity @async_timeout(5.0) diff --git a/tests/unit/benchmark/test_progress.py b/tests/unit/benchmark/test_progress.py index 5c0d98ddf..699696084 100644 --- a/tests/unit/benchmark/test_progress.py +++ b/tests/unit/benchmark/test_progress.py @@ -1,4 +1,7 @@ -"""Unit tests for GenerativeConsoleBenchmarkerProgress refresh helpers.""" +"""Unit tests for GenerativeConsoleBenchmarkerProgress refresh helpers. + +## WRITTEN BY AI ## +""" from unittest.mock import MagicMock, patch @@ -6,6 +9,8 @@ from guidellm.benchmark.progress import GenerativeConsoleBenchmarkerProgress +_MONOTONIC = "guidellm.benchmark.progress.time.monotonic" + @pytest.fixture() def progress(): @@ -14,7 +19,6 @@ def progress(): obj = GenerativeConsoleBenchmarkerProgress.__new__( GenerativeConsoleBenchmarkerProgress ) - # Replicate __init__ state without starting Live obj.display_scheduler_stats = False obj.run_progress = None obj.run_progress_task = None @@ -26,71 +30,73 @@ def progress(): class TestForceRefresh: + @pytest.mark.smoke def test_calls_refresh_immediately(self, progress): + """_force_refresh always calls refresh. ## WRITTEN BY AI ##""" progress._force_refresh() progress.refresh.assert_called_once() + @pytest.mark.sanity def test_resets_last_refresh_timestamp(self, progress): + """_force_refresh records current monotonic time. ## WRITTEN BY AI ##""" fake_now = 123.456 - with patch("guidellm.benchmark.progress.time.monotonic", return_value=fake_now): + with patch(_MONOTONIC, return_value=fake_now): progress._force_refresh() assert progress._last_refresh == fake_now + @pytest.mark.sanity def test_always_refreshes_regardless_of_elapsed(self, progress): + """_force_refresh bypasses the throttle interval. ## WRITTEN BY AI ##""" progress._last_refresh = 9999.0 - with patch( - "guidellm.benchmark.progress.time.monotonic", return_value=9999.001 - ): + with patch(_MONOTONIC, return_value=9999.001): progress._force_refresh() progress.refresh.assert_called_once() class TestThrottledRefresh: + @pytest.mark.smoke def test_refreshes_when_interval_elapsed(self, progress): + """_throttled_refresh renders after the interval passes. ## WRITTEN BY AI ##""" progress._last_refresh = 0.0 - with patch( - "guidellm.benchmark.progress.time.monotonic", return_value=0.26 - ): + with patch(_MONOTONIC, return_value=0.26): progress._throttled_refresh() progress.refresh.assert_called_once() + @pytest.mark.smoke def test_skips_refresh_when_called_too_soon(self, progress): + """_throttled_refresh is a no-op within the interval. ## WRITTEN BY AI ##""" progress._last_refresh = 0.0 - with patch( - "guidellm.benchmark.progress.time.monotonic", return_value=0.10 - ): + with patch(_MONOTONIC, return_value=0.10): progress._throttled_refresh() progress.refresh.assert_not_called() + @pytest.mark.sanity def test_updates_timestamp_on_refresh(self, progress): + """_throttled_refresh records the new time on render. ## WRITTEN BY AI ##""" fake_now = 5.0 progress._last_refresh = 0.0 - with patch( - "guidellm.benchmark.progress.time.monotonic", return_value=fake_now - ): + with patch(_MONOTONIC, return_value=fake_now): progress._throttled_refresh() assert progress._last_refresh == fake_now + @pytest.mark.sanity def test_does_not_update_timestamp_when_skipped(self, progress): + """_throttled_refresh leaves _last_refresh unchanged when skipped. ## WRITTEN BY AI ##""" progress._last_refresh = 10.0 - with patch( - "guidellm.benchmark.progress.time.monotonic", return_value=10.05 - ): + with patch(_MONOTONIC, return_value=10.05): progress._throttled_refresh() assert progress._last_refresh == 10.0 + @pytest.mark.sanity def test_respects_refresh_per_second_boundary(self, progress): + """_throttled_refresh fires exactly at the interval boundary. ## WRITTEN BY AI ##""" progress._last_refresh = 0.0 - interval = 1.0 / progress.refresh_per_second # 0.25s + interval = 1.0 / progress.refresh_per_second # 0.25 s - with patch( - "guidellm.benchmark.progress.time.monotonic", return_value=interval - 0.001 - ): + with patch(_MONOTONIC, return_value=interval - 0.001): progress._throttled_refresh() progress.refresh.assert_not_called() - with patch( - "guidellm.benchmark.progress.time.monotonic", return_value=interval - ): + with patch(_MONOTONIC, return_value=interval): progress._throttled_refresh() progress.refresh.assert_called_once() diff --git a/tests/unit/utils/test_terminal.py b/tests/unit/utils/test_terminal.py new file mode 100644 index 000000000..5f2ea8244 --- /dev/null +++ b/tests/unit/utils/test_terminal.py @@ -0,0 +1,91 @@ +"""Unit tests for suppress_worker_stdio(). + +## WRITTEN BY AI ## +""" + +from __future__ import annotations + +import os +import sys + +import pytest + +from guidellm.utils.terminal import suppress_worker_stdio + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX only") +class TestSuppressWorkerStdio: + @pytest.mark.smoke + def test_stdout_writes_are_silenced(self, capsys): + """Writes to sys.stdout inside the context are suppressed. ## WRITTEN BY AI ##""" + with suppress_worker_stdio(): + sys.stdout.write("should not appear\n") + captured = capsys.readouterr() + assert "should not appear" not in captured.out + + @pytest.mark.smoke + def test_stderr_writes_are_silenced(self, capsys): + """Writes to sys.stderr inside the context are suppressed. ## WRITTEN BY AI ##""" + with suppress_worker_stdio(): + sys.stderr.write("should not appear\n") + captured = capsys.readouterr() + assert "should not appear" not in captured.err + + @pytest.mark.sanity + def test_stdout_restored_after_context(self): + """sys.stdout is restored to the original object after exit. ## WRITTEN BY AI ##""" + original = sys.stdout + with suppress_worker_stdio(): + pass + assert sys.stdout is original + + @pytest.mark.sanity + def test_stderr_restored_after_context(self): + """sys.stderr is restored to the original object after exit. ## WRITTEN BY AI ##""" + original = sys.stderr + with suppress_worker_stdio(): + pass + assert sys.stderr is original + + @pytest.mark.sanity + def test_os_fd1_restored_after_context(self, tmp_path): + """OS-level fd 1 points back to the original file after exit. ## WRITTEN BY AI ##""" + probe = tmp_path / "probe.txt" + saved = os.dup(1) + try: + with suppress_worker_stdio(): + pass + # After the context, writing to fd 1 should reach the real stdout, + # not /dev/null. We verify by checking fd 1 is not /dev/null. + fd1_path = os.readlink(f"/proc/self/fd/1") + assert "null" not in fd1_path + except (OSError, NotImplementedError): + # /proc/self/fd is Linux-specific; skip on others + pytest.skip("cannot inspect /proc/self/fd on this platform") + finally: + os.close(saved) + + @pytest.mark.sanity + def test_no_fd_leak(self): + """suppress_worker_stdio closes all internal fds on exit. ## WRITTEN BY AI ##""" + before = set(os.listdir("/proc/self/fd")) if os.path.exists( + "/proc/self/fd" + ) else None + if before is None: + pytest.skip("cannot inspect /proc/self/fd on this platform") + + with suppress_worker_stdio(): + pass + + after = set(os.listdir("/proc/self/fd")) + # Allow for the listdir fd itself; no net new fds should remain. + assert len(after) <= len(before) + 1 + + @pytest.mark.sanity + def test_stdout_still_works_after_exception(self): + """sys.stdout is restored even when the body raises. ## WRITTEN BY AI ##""" + original = sys.stdout + with pytest.raises(ValueError): + with suppress_worker_stdio(): + raise ValueError("boom") + assert sys.stdout is original From 4fb13688aa18c4e610730440e86f57b4b79e8c1c Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Tue, 4 Aug 2026 15:20:28 +0100 Subject: [PATCH 20/20] Fix offline batch waiter hangs and spawn worker pickling. Defer asyncio lock creation to process_startup() and strip locks in pickle state so spawn workers can unpickle the backend. Route all _run_generate() failures through _signal_batch_failure() so every batch waiter is unblocked on any exception. Co-authored-by: Cursor Signed-off-by: Maryam Tahhan --- src/guidellm/backends/vllm_python/offline.py | 89 ++++++++++------- .../backends/vllm_python/test_vllm_offline.py | 97 ++++++++++++++++++- 2 files changed, 152 insertions(+), 34 deletions(-) diff --git a/src/guidellm/backends/vllm_python/offline.py b/src/guidellm/backends/vllm_python/offline.py index e04a75c4d..8d84c59d7 100644 --- a/src/guidellm/backends/vllm_python/offline.py +++ b/src/guidellm/backends/vllm_python/offline.py @@ -43,6 +43,8 @@ __all__ = ["VLLMOfflineBackend", "VLLMOfflineBackendArgs"] +_ASYNC_LOCK_ATTRS = ("_batch_lock", "_generate_lock", "_engine_lock") + @BackendArgs.register("vllm_offline") class VLLMOfflineBackendArgs(VLLMPythonBackendArgs): @@ -159,15 +161,36 @@ def __init__( """ super().__init__(arguments) - # Batch processing state - self._batch_lock = asyncio.Lock() - self._generate_lock = asyncio.Lock() + # Batch processing state. Asyncio locks are created in + # process_startup() so the backend remains pickleable for spawn + # workers (locks are bound to the parent event loop). + self._batch_lock: asyncio.Lock | None = None + self._generate_lock: asyncio.Lock | None = None self._pending_batch: list[_BatchedRequest] = [] self._processing_task: asyncio.Task[None] | None = None self._shutting_down = False # The synchronous vLLM LLM engine (set during startup) self._llm: Any = None # vllm.LLM + self._engine_lock: asyncio.Lock | None = None + + def __getstate__(self) -> dict[str, Any]: + """Omit asyncio locks so spawn workers can pickle the backend.""" + state = self.__dict__.copy() + for attr in _ASYNC_LOCK_ATTRS: + state.pop(attr, None) + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + """Restore backend state after unpickling in a worker process.""" + self.__dict__.update(state) + for attr in _ASYNC_LOCK_ATTRS: + setattr(self, attr, None) + + def _create_async_locks(self) -> None: + """Create asyncio locks for the current event loop.""" + self._batch_lock = asyncio.Lock() + self._generate_lock = asyncio.Lock() self._engine_lock = asyncio.Lock() # ------------------------------------------------------------------ @@ -185,10 +208,11 @@ async def process_startup(self): double-startup that wastes resources reloading model weights and can cause CPU-affinity degradation. - Asyncio primitives are recreated here because the benchmark - framework forks worker processes after a parent - validate/shutdown cycle; locks created on the parent's - event loop are unusable in the child. + Asyncio primitives are recreated here because scheduler + workers are started in a child process (fork or spawn) with + their own event loop. Locks are not created in + ``__init__`` so the backend can be pickled for spawn + workers; they are bound here instead. :raises RuntimeError: If backend is already initialised """ @@ -204,10 +228,8 @@ async def process_startup(self): # The worker must create its own via _ensure_engine(). self._llm = None - # Recreate asyncio primitives for the current event loop. - self._batch_lock = asyncio.Lock() - self._generate_lock = asyncio.Lock() - self._engine_lock = asyncio.Lock() + # Bind asyncio primitives to the current event loop. + self._create_async_locks() self._pending_batch = [] self._processing_task = None @@ -381,6 +403,22 @@ def _resolve_request( # type: ignore[override] # Batch processing # ------------------------------------------------------------------ + def _signal_batch_failure( + self, + batch: list[_BatchedRequest], + exc: BaseException, + ) -> None: + """Set *exc* on every waiter and unblock ``resolve()`` callers.""" + logger.error( + "vLLM offline batch failed: {}: {}", + type(exc).__name__, + exc, + ) + for batched_req in batch: + if not batched_req.ready.is_set(): + batched_req.result = exc + batched_req.ready.set() + def _take_pending_batch(self) -> list[_BatchedRequest]: """Snapshot and clear ``_pending_batch``. @@ -433,35 +471,20 @@ async def _run_generate(self, batch: list[_BatchedRequest]) -> None: use_tqdm=False, ), ) - except ( - RuntimeError, - ValueError, - TypeError, - OSError, - KeyError, - ) as exc: - logger.error( - "vLLM offline generate() failed: {}: {}", - type(exc).__name__, - exc, - ) - for batched_req in batch: - batched_req.result = exc - batched_req.ready.set() + except Exception as exc: + self._signal_batch_failure(batch, exc) return # Distribute results back to callers. # Use strict=False with explicit length check: a strict zip that raises # ValueError would leave the remaining waiters blocked indefinitely. if len(outputs) != len(batch): - exc = RuntimeError( - f"vLLM returned {len(outputs)} outputs for {len(batch)} requests" + self._signal_batch_failure( + batch, + RuntimeError( + f"vLLM returned {len(outputs)} outputs for {len(batch)} requests" + ), ) - logger.error("{}", exc) - for batched_req in batch: - if not batched_req.ready.is_set(): - batched_req.result = exc - batched_req.ready.set() return for batched_req, output in zip(batch, outputs): diff --git a/tests/unit/backends/vllm_python/test_vllm_offline.py b/tests/unit/backends/vllm_python/test_vllm_offline.py index 3582723ae..715644f65 100644 --- a/tests/unit/backends/vllm_python/test_vllm_offline.py +++ b/tests/unit/backends/vllm_python/test_vllm_offline.py @@ -8,6 +8,7 @@ from __future__ import annotations import asyncio +import pickle import threading from types import SimpleNamespace from unittest.mock import MagicMock, Mock, patch @@ -82,6 +83,7 @@ def offline_backend(): @pytest.fixture def started_backend(offline_backend): """Offline backend with _in_process=True and a mock LLM engine.""" + offline_backend._create_async_locks() offline_backend._in_process = True mock_llm = Mock() mock_llm.generate.return_value = [] @@ -304,6 +306,20 @@ async def test_validate_fails_before_startup(self, offline_backend): class TestProcessStartupReset: + @pytest.mark.smoke + def test_init_does_not_create_async_locks(self): + """Locks are deferred until process_startup for spawn pickling. ## WRITTEN BY AI ##""" + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + ): + backend = _make_offline_backend(model="test-model") + assert backend._batch_lock is None + assert backend._generate_lock is None + assert backend._engine_lock is None + @pytest.mark.asyncio @pytest.mark.smoke async def test_startup_recreates_locks(self): @@ -350,13 +366,75 @@ async def test_startup_clears_llm(self): patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), ): backend = _make_offline_backend(model="test-model") + await backend.process_startup() backend._llm = Mock() - backend._in_process = True await backend.process_shutdown() await backend.process_startup() assert backend._llm is None +class TestSpawnPickling: + @pytest.mark.smoke + def test_backend_pickles_without_async_locks(self): + """Spawn workers can pickle the backend before process_startup. ## WRITTEN BY AI ##""" + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + ): + backend = _make_offline_backend(model="test-model") + data = pickle.dumps(backend) + restored = pickle.loads(data) + + assert restored._batch_lock is None + assert restored._generate_lock is None + assert restored._engine_lock is None + + @pytest.mark.asyncio + @pytest.mark.sanity + async def test_process_startup_after_unpickle_creates_locks(self): + """Unpickled workers bind fresh locks in process_startup. ## WRITTEN BY AI ##""" + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + ): + backend = _make_offline_backend(model="test-model") + restored = pickle.loads(pickle.dumps(backend)) + await restored.process_startup() + + assert restored._batch_lock is not None + assert restored._generate_lock is not None + assert restored._engine_lock is not None + async with restored._batch_lock: + pass + async with restored._generate_lock: + pass + async with restored._engine_lock: + pass + + @pytest.mark.asyncio + @pytest.mark.sanity + async def test_pickles_after_parent_startup_shutdown_cycle(self): + """Parent validate cycle leaves backend spawn-pickleable. ## WRITTEN BY AI ##""" + mock_vllm = MagicMock() + mock_vllm.SamplingParams = _fake_sampling_params + with ( + patch("guidellm.backends.vllm_python.offline.vllm", mock_vllm), + patch("guidellm.backends.vllm_python.vllm.vllm", mock_vllm), + ): + backend = _make_offline_backend(model="test-model") + await backend.process_startup() + await backend.process_shutdown() + restored = pickle.loads(pickle.dumps(backend)) + await restored.process_startup() + + assert restored._in_process is True + assert restored._batch_lock is not None + + # ------------------------------------------------------------------ # Worker-based engine preload # ------------------------------------------------------------------ @@ -508,6 +586,23 @@ async def test_run_generate_output_count_mismatch_signals_all_waiters( assert isinstance(req.result, RuntimeError) assert req.ready.is_set() + @pytest.mark.asyncio + @pytest.mark.sanity + @async_timeout(5.0) + async def test_run_generate_unexpected_exception_signals_all_waiters( + self, started_backend + ): + """Any generate() failure signals all waiters so resolve() never hangs. ## WRITTEN BY AI ##""" + reqs = [ + _BatchedRequest(resolved_prompt="p", multi_modal_data=None, max_tokens=10) + for _ in range(2) + ] + started_backend._llm.generate.side_effect = AttributeError("missing generate") + await started_backend._run_generate(reqs) + for req in reqs: + assert isinstance(req.result, AttributeError) + assert req.ready.is_set() + @pytest.mark.asyncio @pytest.mark.sanity @async_timeout(5.0)