From ffedece7bc093dcd18e4259c0d753f3cf03d7b27 Mon Sep 17 00:00:00 2001 From: Uri Shaket Date: Mon, 2 Mar 2026 23:55:49 +0200 Subject: [PATCH 1/7] mvoe validation logic to RequestHandlers Signed-off-by: Uri Shaket --- .../backends/openai/request_handlers.py | 26 +++- .../backends/openai/test_request_handlers.py | 116 ++++++++++++------ 2 files changed, 103 insertions(+), 39 deletions(-) diff --git a/src/guidellm/backends/openai/request_handlers.py b/src/guidellm/backends/openai/request_handlers.py index 1a0451682..11bb57a27 100644 --- a/src/guidellm/backends/openai/request_handlers.py +++ b/src/guidellm/backends/openai/request_handlers.py @@ -512,7 +512,7 @@ def compile_non_streaming( text = choice.get("text", "") input_metrics, output_metrics = self.extract_metrics(usage, text) - return GenerationResponse( + compiled = GenerationResponse( request_id=request.request_id, request_args=arguments.model_dump_json(), response_id=response.get("id"), # use vLLM ID if available @@ -520,6 +520,8 @@ def compile_non_streaming( input_metrics=input_metrics, output_metrics=output_metrics, ) + self._validate_compiled_response(compiled) + return compiled def add_streaming_line(self, line: str) -> int | None: """ @@ -562,7 +564,7 @@ def compile_streaming( text = "".join(self.streaming_texts) input_metrics, output_metrics = self.extract_metrics(self.streaming_usage, text) - return GenerationResponse( + compiled = GenerationResponse( request_id=request.request_id, request_args=arguments.model_dump_json(), response_id=self.streaming_response_id, # use vLLM ID if available @@ -570,6 +572,18 @@ def compile_streaming( input_metrics=input_metrics, output_metrics=output_metrics, ) + self._validate_compiled_response(compiled) + return compiled + + def _validate_compiled_response(self, response: GenerationResponse) -> None: + """Raise when endpoint produced a terminal payload with no usable output.""" + has_text = bool(response.text and response.text.strip()) + output_tokens = response.output_metrics.total_tokens or 0 + if not has_text and output_tokens <= 0: + raise ValueError( + "[UNUSABLE_BACKEND_RESPONSE] backend resolved without a usable " + "terminal response payload" + ) def extract_line_data(self, line: str) -> dict[str, Any] | None: """ @@ -1074,7 +1088,7 @@ def compile_non_streaming( output_metrics, len(tool_calls) if tool_calls else 0, text ) - return GenerationResponse( + compiled = GenerationResponse( request_id=request.request_id, request_args=arguments.model_dump_json(), response_id=response.get("id"), # use vLLM ID if available @@ -1084,6 +1098,8 @@ def compile_non_streaming( input_metrics=input_metrics, output_metrics=output_metrics, ) + self._validate_compiled_response(compiled) + return compiled def add_streaming_line(self, line: str) -> int | None: """ @@ -1175,7 +1191,7 @@ def compile_streaming( :param request: Original generation request :return: Standardized GenerationResponse with concatenated content and metrics """ - return _compile_streaming_response( + compiled = _compile_streaming_response( request, arguments, self.streaming_texts, @@ -1185,6 +1201,8 @@ def compile_streaming( self.extract_metrics, streaming_reasoning_texts=self.streaming_reasoning_texts, ) + self._validate_compiled_response(compiled) + return compiled @OpenAIRequestHandlerFactory.register( diff --git a/tests/unit/backends/openai/test_request_handlers.py b/tests/unit/backends/openai/test_request_handlers.py index 287b22413..c3e25f6bc 100644 --- a/tests/unit/backends/openai/test_request_handlers.py +++ b/tests/unit/backends/openai/test_request_handlers.py @@ -575,9 +575,6 @@ def test_format_ignore_eos(self, valid_instances): 10, 5, ), - ({"choices": [{"text": ""}], "usage": {}}, "", None, None), - ({"choices": [], "usage": {}}, "", None, None), - ({}, "", None, None), ], ) def test_non_streaming( @@ -638,7 +635,6 @@ def test_non_streaming( None, None, ), - (["", "data: [DONE]"], "", None, None), ], ) def test_streaming( @@ -672,6 +668,47 @@ def test_streaming( assert response.output_metrics.text_words == len(expected_text.split()) assert response.output_metrics.text_characters == len(expected_text) + @pytest.mark.regression + @pytest.mark.parametrize( + "response", + [ + {"choices": [{"text": ""}], "usage": {}}, + {"choices": [], "usage": {}}, + {}, + ], + ) + def test_non_streaming_raises_for_unusable_terminal_payload( + self, valid_instances, generation_request, response + ): + """Test unusable non-streaming text response raises. + + ### WRITTEN BY AI ### + """ + instance = valid_instances + arguments = instance.format(generation_request) + + with pytest.raises(ValueError, match="UNUSABLE_BACKEND_RESPONSE"): + instance.compile_non_streaming(generation_request, arguments, response) + + @pytest.mark.regression + def test_streaming_raises_for_unusable_terminal_payload( + self, valid_instances, generation_request + ): + """Test unusable streaming text response raises. + + ### WRITTEN BY AI ### + """ + instance = valid_instances + arguments = instance.format(generation_request) + + for line in ["", "data: [DONE]"]: + result = instance.add_streaming_line(line) + if result is None: + break + + with pytest.raises(ValueError, match="UNUSABLE_BACKEND_RESPONSE"): + instance.compile_streaming(generation_request, arguments) + @pytest.mark.smoke @pytest.mark.parametrize( ("line", "expected_output"), @@ -1112,18 +1149,6 @@ def test_format_multimodal(self, valid_instances): 10, 5, ), - ( - {"choices": [{"message": {"content": ""}}], "usage": {}}, - "", - None, - None, - ), - ( - {"choices": [], "usage": {}}, - "", - None, - None, - ), ], ) def test_non_streaming( @@ -1177,12 +1202,6 @@ def test_non_streaming( None, None, ), - ( - ["", "data: [DONE]"], - "", - None, - None, - ), ], ) def test_streaming( @@ -1214,6 +1233,46 @@ def test_streaming( assert response.input_metrics.text_tokens == expected_input_tokens assert response.output_metrics.text_tokens == expected_output_tokens + @pytest.mark.regression + @pytest.mark.parametrize( + "response", + [ + {"choices": [{"message": {"content": ""}}], "usage": {}}, + {"choices": [], "usage": {}}, + ], + ) + def test_non_streaming_raises_for_unusable_terminal_payload( + self, valid_instances, generation_request, response + ): + """Test unusable non-streaming chat response raises. + + ### WRITTEN BY AI ### + """ + instance = valid_instances + arguments = instance.format(generation_request) + + with pytest.raises(ValueError, match="UNUSABLE_BACKEND_RESPONSE"): + instance.compile_non_streaming(generation_request, arguments, response) + + @pytest.mark.regression + def test_streaming_raises_for_unusable_terminal_payload( + self, valid_instances, generation_request + ): + """Test unusable streaming chat response raises. + + ### WRITTEN BY AI ### + """ + instance = valid_instances + arguments = instance.format(generation_request) + + for line in ["", "data: [DONE]"]: + result = instance.add_streaming_line(line) + if result is None: + break + + with pytest.raises(ValueError, match="UNUSABLE_BACKEND_RESPONSE"): + instance.compile_streaming(generation_request, arguments) + @pytest.mark.sanity def test_streaming_reasoning_tokens(self, valid_instances, generation_request): """Test that reasoning tokens are properly detected for TTFT measurement. @@ -1228,15 +1287,12 @@ def test_streaming_reasoning_tokens(self, valid_instances, generation_request): arguments = instance.format(generation_request) lines = [ - # First chunk has reasoning token ( 'data: {"id": "chatcmpl-123", "choices": ' '[{"index": 0, "delta": {"reasoning": "Okay"}}], "usage": {}}' ), - # More reasoning tokens 'data: {"choices": [{"delta": {"reasoning": ", let me"}}], "usage": {}}', 'data: {"choices": [{"delta": {"reasoning": " think..."}}], "usage": {}}', - # Finally content tokens 'data: {"choices": [{"delta": {"content": "Hello"}}], "usage": {}}', ( 'data: {"choices": [{"delta": {"content": " world!"}}], ' @@ -1256,17 +1312,13 @@ def test_streaming_reasoning_tokens(self, valid_instances, generation_request): elif result is None: break - # Verify that the first update happened on the first reasoning token (line 0) assert first_update_on_line == 0, ( f"Expected first token detection on line 0 (reasoning token), " f"but got {first_update_on_line}" ) - - # Verify all chunks with content were counted (5 lines with tokens) assert updated_count == 5 response = instance.compile_streaming(generation_request, arguments) - # Reasoning tokens should NOT appear in response.text; only content does assert "Okay" not in response.text assert "let me think..." not in response.text assert response.text == "Hello world!" @@ -1289,7 +1341,6 @@ def test_streaming_both_reasoning_and_content_in_same_chunk( arguments = instance.format(generation_request) lines = [ - # Chunk with both reasoning and content (edge case) ( 'data: {"choices": [{"delta": ' '{"reasoning": "Let me think...", "content": "Answer: "}}], ' @@ -1307,12 +1358,9 @@ def test_streaming_both_reasoning_and_content_in_same_chunk( if result > 0: updated_count += 1 - # First chunk has both reasoning and content (counts as 1 iteration) - # Second chunk has content only (counts as 1 iteration) assert updated_count == 2 response = instance.compile_streaming(generation_request, arguments) - # Reasoning text should NOT appear; only content is captured assert "Let me think..." not in response.text assert response.text == "Answer: 42" @@ -2065,8 +2113,6 @@ def test_format_strips_tool_choice_without_tools(self, valid_instances): assert "tool_choice" not in result.body assert "tools" not in result.body - - class TestAudioRequestHandler: """Test cases for AudioRequestHandler. From 03cdec7a6954e3dcaa610a9c5b382966a8212098 Mon Sep 17 00:00:00 2001 From: Uri Shaket Date: Mon, 2 Mar 2026 23:57:34 +0200 Subject: [PATCH 2/7] revert worker changes Signed-off-by: Uri Shaket --- src/guidellm/scheduler/worker.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/guidellm/scheduler/worker.py b/src/guidellm/scheduler/worker.py index 2df81b110..5c864ec75 100644 --- a/src/guidellm/scheduler/worker.py +++ b/src/guidellm/scheduler/worker.py @@ -407,18 +407,11 @@ async def _process_next_request( # noqa: C901 async for resp, info in self.backend.resolve( # type: ignore[attr-defined] request, request_info, history or None ): + response = resp request_info = info if request_info is None: raise RuntimeError("Received invalid request info from backend") - if ( - resp is None - and request_info.timings.first_token_iteration is not None - ): - self._send_update("first_token", None, request, request_info) - - response = resp - # Complete the request request_info.timings.resolve_end = time.time() self._send_update("completed", response, request, request_info) From e5fda3292f86e3f7892171a4106fb415504b66df Mon Sep 17 00:00:00 2001 From: Uri Shaket Date: Wed, 15 Jul 2026 11:25:12 +0300 Subject: [PATCH 3/7] fix tests Signed-off-by: Uri Shaket --- tests/unit/backends/openai/test_http.py | 6 +++++- tests/unit/backends/openai/test_request_handlers.py | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/unit/backends/openai/test_http.py b/tests/unit/backends/openai/test_http.py index d053699ce..0d8606517 100644 --- a/tests/unit/backends/openai/test_http.py +++ b/tests/unit/backends/openai/test_http.py @@ -449,7 +449,11 @@ def mock_fail(*args, **kwargs): @async_timeout(10.0) async def test_resolve_with_history(self, httpx_mock: HTTPXMock): """Test resolve method handles conversation history.""" - backend = _make_backend(target="http://test", request_format="/v1/completions") + backend = _make_backend( + target="http://test", + request_format="/v1/completions", + stream=False, + ) # Mock the models endpoint httpx_mock.add_response( diff --git a/tests/unit/backends/openai/test_request_handlers.py b/tests/unit/backends/openai/test_request_handlers.py index c3e25f6bc..8e5129c92 100644 --- a/tests/unit/backends/openai/test_request_handlers.py +++ b/tests/unit/backends/openai/test_request_handlers.py @@ -2113,6 +2113,8 @@ def test_format_strips_tool_choice_without_tools(self, valid_instances): assert "tool_choice" not in result.body assert "tools" not in result.body + + class TestAudioRequestHandler: """Test cases for AudioRequestHandler. From d459b2ef7eeda1bf1b0b8f0cf3655173163a2520 Mon Sep 17 00:00:00 2001 From: Uri Shaket Date: Wed, 15 Jul 2026 13:23:05 +0300 Subject: [PATCH 4/7] self CR Signed-off-by: Uri Shaket --- src/guidellm/scheduler/worker.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/guidellm/scheduler/worker.py b/src/guidellm/scheduler/worker.py index 5c864ec75..2df81b110 100644 --- a/src/guidellm/scheduler/worker.py +++ b/src/guidellm/scheduler/worker.py @@ -407,11 +407,18 @@ async def _process_next_request( # noqa: C901 async for resp, info in self.backend.resolve( # type: ignore[attr-defined] request, request_info, history or None ): - response = resp request_info = info if request_info is None: raise RuntimeError("Received invalid request info from backend") + if ( + resp is None + and request_info.timings.first_token_iteration is not None + ): + self._send_update("first_token", None, request, request_info) + + response = resp + # Complete the request request_info.timings.resolve_end = time.time() self._send_update("completed", response, request, request_info) From 36036d0b0b7f67c111337f628e64173dfd5e69fc Mon Sep 17 00:00:00 2001 From: Uri Shaket Date: Thu, 16 Jul 2026 17:28:58 +0300 Subject: [PATCH 5/7] Clarify unusable backend response error message Signed-off-by: Uri Shaket Co-authored-by: Cursor --- src/guidellm/backends/openai/request_handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/guidellm/backends/openai/request_handlers.py b/src/guidellm/backends/openai/request_handlers.py index 11bb57a27..06521bc22 100644 --- a/src/guidellm/backends/openai/request_handlers.py +++ b/src/guidellm/backends/openai/request_handlers.py @@ -581,8 +581,8 @@ def _validate_compiled_response(self, response: GenerationResponse) -> None: output_tokens = response.output_metrics.total_tokens or 0 if not has_text and output_tokens <= 0: raise ValueError( - "[UNUSABLE_BACKEND_RESPONSE] backend resolved without a usable " - "terminal response payload" + "[UNUSABLE_BACKEND_RESPONSE] backend resolved with empty " + "response payload" ) def extract_line_data(self, line: str) -> dict[str, Any] | None: From 1789158ca2cedcf63d8ef55927ee77e77358c03a Mon Sep 17 00:00:00 2001 From: Uri Shaket Date: Tue, 21 Jul 2026 20:39:29 +0300 Subject: [PATCH 6/7] refactor: centralize post_validation hook for response validation Add post_validation to OpenAIRequestHandler and OpenAIWSRequestHandler protocols with a permissive no-op default. TextCompletionsRequestHandler and ResponsesRequestHandler override with a check for text, tool_calls, or output_tokens. PoolingRequestHandler and EmbeddingsRequestHandler keep the default no-op since they produce non-text output. Validation is called from the backend resolve methods (http.py, websocket.py) after compile, not inside each handler's compile methods. Cancellation paths skip validation for partial responses. Fixes tool-call-only false negative: responses with tool_calls but no text are now correctly treated as valid output. Assisted-by: Claude Signed-off-by: Uri Shaket Co-authored-by: Cursor --- src/guidellm/backends/openai/http.py | 2 + .../backends/openai/request_handlers.py | 59 +++++-- src/guidellm/backends/openai/websocket.py | 4 +- .../backends/openai/test_request_handlers.py | 153 ++++++++++++------ 4 files changed, 149 insertions(+), 69 deletions(-) diff --git a/src/guidellm/backends/openai/http.py b/src/guidellm/backends/openai/http.py index f6ec2beab..bdba91fd7 100644 --- a/src/guidellm/backends/openai/http.py +++ b/src/guidellm/backends/openai/http.py @@ -483,6 +483,7 @@ async def _resolve_non_streaming( response.raise_for_status() data = response.json() gen_response = request_handler.compile_non_streaming(request, arguments, data) + request_handler.post_validation(gen_response) yield gen_response, request_info self._check_tool_call_expectations(request, gen_response) @@ -553,6 +554,7 @@ async def _resolve_streaming( request_info.timings.request_end = time.time() gen_response = request_handler.compile_streaming(request, arguments) + request_handler.post_validation(gen_response) self._check_tool_call_expectations(request, gen_response) yield gen_response, request_info except asyncio.CancelledError as err: diff --git a/src/guidellm/backends/openai/request_handlers.py b/src/guidellm/backends/openai/request_handlers.py index 06521bc22..7e703237f 100644 --- a/src/guidellm/backends/openai/request_handlers.py +++ b/src/guidellm/backends/openai/request_handlers.py @@ -126,6 +126,18 @@ def compile_streaming( """ ... + def post_validation(self, response: GenerationResponse) -> None: + """Validate a compiled response before returning it. + + Default implementation is permissive (no-op). Handlers override + this to reject responses that lack usable output for their + endpoint type. + + :param response: The compiled generation response to validate. + :raises ValueError: If the response is unusable. + """ + ... + class OpenAIRequestHandlerFactory(RegistryMixin[type[OpenAIRequestHandler]]): """ @@ -272,6 +284,18 @@ def compile_streaming( """ ... + def post_validation(self, response: GenerationResponse) -> None: + """Validate a compiled response before returning it. + + Default implementation is permissive (no-op). Handlers override + this to reject responses that lack usable output for their + endpoint type. + + :param response: The compiled generation response to validate. + :raises ValueError: If the response is unusable. + """ + ... + class OpenAIWSRequestHandlerFactory(RegistryMixin["type[OpenAIWSRequestHandler]"]): """Factory for registering and creating WebSocket request handlers by path.""" @@ -512,7 +536,7 @@ def compile_non_streaming( text = choice.get("text", "") input_metrics, output_metrics = self.extract_metrics(usage, text) - compiled = GenerationResponse( + return GenerationResponse( request_id=request.request_id, request_args=arguments.model_dump_json(), response_id=response.get("id"), # use vLLM ID if available @@ -520,8 +544,6 @@ def compile_non_streaming( input_metrics=input_metrics, output_metrics=output_metrics, ) - self._validate_compiled_response(compiled) - return compiled def add_streaming_line(self, line: str) -> int | None: """ @@ -564,7 +586,7 @@ def compile_streaming( text = "".join(self.streaming_texts) input_metrics, output_metrics = self.extract_metrics(self.streaming_usage, text) - compiled = GenerationResponse( + return GenerationResponse( request_id=request.request_id, request_args=arguments.model_dump_json(), response_id=self.streaming_response_id, # use vLLM ID if available @@ -572,14 +594,12 @@ def compile_streaming( input_metrics=input_metrics, output_metrics=output_metrics, ) - self._validate_compiled_response(compiled) - return compiled - def _validate_compiled_response(self, response: GenerationResponse) -> None: - """Raise when endpoint produced a terminal payload with no usable output.""" + def post_validation(self, response: GenerationResponse) -> None: has_text = bool(response.text and response.text.strip()) + has_tool_calls = bool(response.tool_calls) output_tokens = response.output_metrics.total_tokens or 0 - if not has_text and output_tokens <= 0: + if not has_text and not has_tool_calls and output_tokens <= 0: raise ValueError( "[UNUSABLE_BACKEND_RESPONSE] backend resolved with empty " "response payload" @@ -1088,7 +1108,7 @@ def compile_non_streaming( output_metrics, len(tool_calls) if tool_calls else 0, text ) - compiled = GenerationResponse( + return GenerationResponse( request_id=request.request_id, request_args=arguments.model_dump_json(), response_id=response.get("id"), # use vLLM ID if available @@ -1098,8 +1118,6 @@ def compile_non_streaming( input_metrics=input_metrics, output_metrics=output_metrics, ) - self._validate_compiled_response(compiled) - return compiled def add_streaming_line(self, line: str) -> int | None: """ @@ -1191,7 +1209,7 @@ def compile_streaming( :param request: Original generation request :return: Standardized GenerationResponse with concatenated content and metrics """ - compiled = _compile_streaming_response( + return _compile_streaming_response( request, arguments, self.streaming_texts, @@ -1201,8 +1219,6 @@ def compile_streaming( self.extract_metrics, streaming_reasoning_texts=self.streaming_reasoning_texts, ) - self._validate_compiled_response(compiled) - return compiled @OpenAIRequestHandlerFactory.register( @@ -1866,6 +1882,16 @@ def compile_streaming( streaming_reasoning_texts=self.streaming_reasoning_texts, ) + def post_validation(self, response: GenerationResponse) -> None: + has_text = bool(response.text and response.text.strip()) + has_tool_calls = bool(response.tool_calls) + output_tokens = response.output_metrics.total_tokens or 0 + if not has_text and not has_tool_calls and output_tokens <= 0: + raise ValueError( + "[UNUSABLE_BACKEND_RESPONSE] backend resolved with empty " + "response payload" + ) + def extract_line_data(self, line: str) -> dict[str, Any] | None: """Parse a Responses API SSE line. @@ -2165,6 +2191,9 @@ class PoolingRequestHandler(ChatCompletionsRequestHandler): pooling-specific request structure with nested data fields. """ + def post_validation(self, response: GenerationResponse) -> None: # noqa: ARG002 + pass + def format( self, data: GenerationRequest, diff --git a/src/guidellm/backends/openai/websocket.py b/src/guidellm/backends/openai/websocket.py index 795872233..a6aa21981 100644 --- a/src/guidellm/backends/openai/websocket.py +++ b/src/guidellm/backends/openai/websocket.py @@ -506,7 +506,9 @@ async def resolve( # type: ignore[override, misc] # noqa: C901, PLR0912, PLR09 f"(last type={event.get('type')!r})." ) - yield handler.compile_streaming(request, arguments), request_info + compiled = handler.compile_streaming(request, arguments) + handler.post_validation(compiled) + yield compiled, request_info except asyncio.CancelledError as err: yield handler.compile_streaming(request, arguments), request_info diff --git a/tests/unit/backends/openai/test_request_handlers.py b/tests/unit/backends/openai/test_request_handlers.py index 8e5129c92..5354a92b8 100644 --- a/tests/unit/backends/openai/test_request_handlers.py +++ b/tests/unit/backends/openai/test_request_handlers.py @@ -686,9 +686,12 @@ def test_non_streaming_raises_for_unusable_terminal_payload( """ instance = valid_instances arguments = instance.format(generation_request) + compiled = instance.compile_non_streaming( + generation_request, arguments, response + ) with pytest.raises(ValueError, match="UNUSABLE_BACKEND_RESPONSE"): - instance.compile_non_streaming(generation_request, arguments, response) + instance.post_validation(compiled) @pytest.mark.regression def test_streaming_raises_for_unusable_terminal_payload( @@ -706,8 +709,10 @@ def test_streaming_raises_for_unusable_terminal_payload( if result is None: break + compiled = instance.compile_streaming(generation_request, arguments) + with pytest.raises(ValueError, match="UNUSABLE_BACKEND_RESPONSE"): - instance.compile_streaming(generation_request, arguments) + instance.post_validation(compiled) @pytest.mark.smoke @pytest.mark.parametrize( @@ -1250,9 +1255,12 @@ def test_non_streaming_raises_for_unusable_terminal_payload( """ instance = valid_instances arguments = instance.format(generation_request) + compiled = instance.compile_non_streaming( + generation_request, arguments, response + ) with pytest.raises(ValueError, match="UNUSABLE_BACKEND_RESPONSE"): - instance.compile_non_streaming(generation_request, arguments, response) + instance.post_validation(compiled) @pytest.mark.regression def test_streaming_raises_for_unusable_terminal_payload( @@ -1270,8 +1278,45 @@ def test_streaming_raises_for_unusable_terminal_payload( if result is None: break + compiled = instance.compile_streaming(generation_request, arguments) + with pytest.raises(ValueError, match="UNUSABLE_BACKEND_RESPONSE"): - instance.compile_streaming(generation_request, arguments) + instance.post_validation(compiled) + + @pytest.mark.regression + def test_non_streaming_tool_call_only_passes_validation( + self, valid_instances, generation_request + ): + """Tool-call-only response (no text) is valid and must not raise. + + ### WRITTEN BY AI ### + """ + instance = valid_instances + arguments = instance.format(generation_request) + response = { + "choices": [ + { + "message": { + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city":"NYC"}', + }, + } + ], + } + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5}, + } + result = instance.compile_non_streaming(generation_request, arguments, response) + instance.post_validation(result) + assert result.tool_calls is not None + assert len(result.tool_calls) == 1 @pytest.mark.sanity def test_streaming_reasoning_tokens(self, valid_instances, generation_request): @@ -3050,18 +3095,6 @@ def test_format_input_items_image(self, valid_instances): 10, 8, ), - ( - {"id": "resp_789", "output": [], "usage": {}}, - "", - None, - None, - ), - ( - {"output": []}, - "", - None, - None, - ), ], ) def test_non_streaming( @@ -3155,29 +3188,6 @@ def test_non_streaming( None, None, ), - ( - [ - "event: response.created", - ( - "data: {" - '"type":"response.created",' - '"response":{"id":"resp_3"},' - '"sequence_number":0}' - ), - "", - "event: response.completed", - ( - "data: {" - '"type":"response.completed",' - '"response":{"id":"resp_3","usage":{}},' - '"sequence_number":2}' - ), - "data: [DONE]", - ], - "", - None, - None, - ), ], ) def test_streaming( @@ -3337,20 +3347,6 @@ def test_extract_line_data(self, valid_instances, line, expected_output): 10, 2, ), - ( - [ - "event: response.failed", - ( - "data: {" - '"type":"response.failed",' - '"response":{"id":"resp_fail_no_usage"},' - '"sequence_number":1}' - ), - ], - "", - None, - None, - ), ], ) def test_streaming_terminal_events( @@ -3380,6 +3376,57 @@ def test_streaming_terminal_events( assert response.input_metrics.text_tokens == expected_input_tokens assert response.output_metrics.text_tokens == expected_output_tokens + @pytest.mark.regression + @pytest.mark.parametrize( + "response", + [ + {"output": [], "usage": {}}, + {"output": [{"type": "message", "content": []}], "usage": {}}, + ], + ) + def test_non_streaming_raises_for_unusable_terminal_payload( + self, valid_instances, generation_request, response + ): + """Responses API empty output raises UNUSABLE_BACKEND_RESPONSE. + + ### WRITTEN BY AI ### + """ + instance = valid_instances + arguments = instance.format(generation_request) + + compiled = instance.compile_non_streaming( + generation_request, arguments, response + ) + + with pytest.raises(ValueError, match="UNUSABLE_BACKEND_RESPONSE"): + instance.post_validation(compiled) + + @pytest.mark.regression + def test_non_streaming_tool_call_only_passes_validation( + self, valid_instances, generation_request + ): + """Tool-call-only Responses API output is valid and must not raise. + + ### WRITTEN BY AI ### + """ + instance = valid_instances + arguments = instance.format(generation_request) + response = { + "output": [ + { + "type": "function_call", + "call_id": "call_1", + "name": "get_weather", + "arguments": '{"city":"NYC"}', + } + ], + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + result = instance.compile_non_streaming(generation_request, arguments, response) + instance.post_validation(result) + assert result.tool_calls is not None + assert len(result.tool_calls) == 1 + @pytest.mark.sanity def test_streaming_reasoning_triggers_ttft_not_content( self, valid_instances, generation_request From 6de940a469e2bd55cbcacf96cef81d7a23c7047e Mon Sep 17 00:00:00 2001 From: Uri Shaket Date: Thu, 23 Jul 2026 09:44:58 +0300 Subject: [PATCH 7/7] refactor: deduplicate post_validation and add docstrings Extract _validate_text_response helper shared by TextCompletionsRequestHandler and ResponsesRequestHandler. Add docstrings to all post_validation overrides. Assisted-by: Claude Signed-off-by: Uri Shaket Co-authored-by: Cursor --- .../backends/openai/request_handlers.py | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/guidellm/backends/openai/request_handlers.py b/src/guidellm/backends/openai/request_handlers.py index 7e703237f..6e67ee26f 100644 --- a/src/guidellm/backends/openai/request_handlers.py +++ b/src/guidellm/backends/openai/request_handlers.py @@ -341,6 +341,24 @@ def _apply_tool_call_metrics( output_metrics.mixed_content_tool_tokens = output_metrics.text_tokens +def _validate_text_response(response: GenerationResponse) -> None: + """Reject a compiled response that has no usable output. + + A response is considered usable if it has non-empty text, tool calls, + or output tokens. Used by text/chat completions and responses handlers. + + :param response: The compiled generation response to validate. + :raises ValueError: If the response contains no usable output. + """ + has_text = bool(response.text and response.text.strip()) + has_tool_calls = bool(response.tool_calls) + output_tokens = response.output_metrics.total_tokens or 0 + if not has_text and not has_tool_calls and output_tokens <= 0: + raise ValueError( + "[UNUSABLE_BACKEND_RESPONSE] backend resolved with empty response payload" + ) + + _DEFAULT_REASONING_TEMPLATE = "{reasoning}" @@ -596,14 +614,8 @@ def compile_streaming( ) def post_validation(self, response: GenerationResponse) -> None: - has_text = bool(response.text and response.text.strip()) - has_tool_calls = bool(response.tool_calls) - output_tokens = response.output_metrics.total_tokens or 0 - if not has_text and not has_tool_calls and output_tokens <= 0: - raise ValueError( - "[UNUSABLE_BACKEND_RESPONSE] backend resolved with empty " - "response payload" - ) + """Reject responses with no text, tool calls, or output tokens.""" + _validate_text_response(response) def extract_line_data(self, line: str) -> dict[str, Any] | None: """ @@ -1883,14 +1895,8 @@ def compile_streaming( ) def post_validation(self, response: GenerationResponse) -> None: - has_text = bool(response.text and response.text.strip()) - has_tool_calls = bool(response.tool_calls) - output_tokens = response.output_metrics.total_tokens or 0 - if not has_text and not has_tool_calls and output_tokens <= 0: - raise ValueError( - "[UNUSABLE_BACKEND_RESPONSE] backend resolved with empty " - "response payload" - ) + """Reject responses with no text, tool calls, or output tokens.""" + _validate_text_response(response) def extract_line_data(self, line: str) -> dict[str, Any] | None: """Parse a Responses API SSE line. @@ -2192,7 +2198,7 @@ class PoolingRequestHandler(ChatCompletionsRequestHandler): """ def post_validation(self, response: GenerationResponse) -> None: # noqa: ARG002 - pass + """Pooling responses produce non-text output; skip validation.""" def format( self,