diff --git a/src/guidellm/backends/openai/http.py b/src/guidellm/backends/openai/http.py index f6ec2beab..bdba91fd7 100644 --- a/src/guidellm/backends/openai/http.py +++ b/src/guidellm/backends/openai/http.py @@ -483,6 +483,7 @@ async def _resolve_non_streaming( response.raise_for_status() data = response.json() gen_response = request_handler.compile_non_streaming(request, arguments, data) + request_handler.post_validation(gen_response) yield gen_response, request_info self._check_tool_call_expectations(request, gen_response) @@ -553,6 +554,7 @@ async def _resolve_streaming( request_info.timings.request_end = time.time() gen_response = request_handler.compile_streaming(request, arguments) + request_handler.post_validation(gen_response) self._check_tool_call_expectations(request, gen_response) yield gen_response, request_info except asyncio.CancelledError as err: diff --git a/src/guidellm/backends/openai/request_handlers.py b/src/guidellm/backends/openai/request_handlers.py index 1a0451682..6e67ee26f 100644 --- a/src/guidellm/backends/openai/request_handlers.py +++ b/src/guidellm/backends/openai/request_handlers.py @@ -126,6 +126,18 @@ def compile_streaming( """ ... + def post_validation(self, response: GenerationResponse) -> None: + """Validate a compiled response before returning it. + + Default implementation is permissive (no-op). Handlers override + this to reject responses that lack usable output for their + endpoint type. + + :param response: The compiled generation response to validate. + :raises ValueError: If the response is unusable. + """ + ... + class OpenAIRequestHandlerFactory(RegistryMixin[type[OpenAIRequestHandler]]): """ @@ -272,6 +284,18 @@ def compile_streaming( """ ... + def post_validation(self, response: GenerationResponse) -> None: + """Validate a compiled response before returning it. + + Default implementation is permissive (no-op). Handlers override + this to reject responses that lack usable output for their + endpoint type. + + :param response: The compiled generation response to validate. + :raises ValueError: If the response is unusable. + """ + ... + class OpenAIWSRequestHandlerFactory(RegistryMixin["type[OpenAIWSRequestHandler]"]): """Factory for registering and creating WebSocket request handlers by path.""" @@ -317,6 +341,24 @@ def _apply_tool_call_metrics( output_metrics.mixed_content_tool_tokens = output_metrics.text_tokens +def _validate_text_response(response: GenerationResponse) -> None: + """Reject a compiled response that has no usable output. + + A response is considered usable if it has non-empty text, tool calls, + or output tokens. Used by text/chat completions and responses handlers. + + :param response: The compiled generation response to validate. + :raises ValueError: If the response contains no usable output. + """ + has_text = bool(response.text and response.text.strip()) + has_tool_calls = bool(response.tool_calls) + output_tokens = response.output_metrics.total_tokens or 0 + if not has_text and not has_tool_calls and output_tokens <= 0: + raise ValueError( + "[UNUSABLE_BACKEND_RESPONSE] backend resolved with empty response payload" + ) + + _DEFAULT_REASONING_TEMPLATE = "{reasoning}" @@ -571,6 +613,10 @@ def compile_streaming( output_metrics=output_metrics, ) + def post_validation(self, response: GenerationResponse) -> None: + """Reject responses with no text, tool calls, or output tokens.""" + _validate_text_response(response) + def extract_line_data(self, line: str) -> dict[str, Any] | None: """ Extract JSON data from a streaming response line. @@ -1848,6 +1894,10 @@ def compile_streaming( streaming_reasoning_texts=self.streaming_reasoning_texts, ) + def post_validation(self, response: GenerationResponse) -> None: + """Reject responses with no text, tool calls, or output tokens.""" + _validate_text_response(response) + def extract_line_data(self, line: str) -> dict[str, Any] | None: """Parse a Responses API SSE line. @@ -2147,6 +2197,9 @@ class PoolingRequestHandler(ChatCompletionsRequestHandler): pooling-specific request structure with nested data fields. """ + def post_validation(self, response: GenerationResponse) -> None: # noqa: ARG002 + """Pooling responses produce non-text output; skip validation.""" + def format( self, data: GenerationRequest, diff --git a/src/guidellm/backends/openai/websocket.py b/src/guidellm/backends/openai/websocket.py index 795872233..a6aa21981 100644 --- a/src/guidellm/backends/openai/websocket.py +++ b/src/guidellm/backends/openai/websocket.py @@ -506,7 +506,9 @@ async def resolve( # type: ignore[override, misc] # noqa: C901, PLR0912, PLR09 f"(last type={event.get('type')!r})." ) - yield handler.compile_streaming(request, arguments), request_info + compiled = handler.compile_streaming(request, arguments) + handler.post_validation(compiled) + yield compiled, request_info except asyncio.CancelledError as err: yield handler.compile_streaming(request, arguments), request_info diff --git a/tests/unit/backends/openai/test_http.py b/tests/unit/backends/openai/test_http.py index d053699ce..0d8606517 100644 --- a/tests/unit/backends/openai/test_http.py +++ b/tests/unit/backends/openai/test_http.py @@ -449,7 +449,11 @@ def mock_fail(*args, **kwargs): @async_timeout(10.0) async def test_resolve_with_history(self, httpx_mock: HTTPXMock): """Test resolve method handles conversation history.""" - backend = _make_backend(target="http://test", request_format="/v1/completions") + backend = _make_backend( + target="http://test", + request_format="/v1/completions", + stream=False, + ) # Mock the models endpoint httpx_mock.add_response( diff --git a/tests/unit/backends/openai/test_request_handlers.py b/tests/unit/backends/openai/test_request_handlers.py index 287b22413..5354a92b8 100644 --- a/tests/unit/backends/openai/test_request_handlers.py +++ b/tests/unit/backends/openai/test_request_handlers.py @@ -575,9 +575,6 @@ def test_format_ignore_eos(self, valid_instances): 10, 5, ), - ({"choices": [{"text": ""}], "usage": {}}, "", None, None), - ({"choices": [], "usage": {}}, "", None, None), - ({}, "", None, None), ], ) def test_non_streaming( @@ -638,7 +635,6 @@ def test_non_streaming( None, None, ), - (["", "data: [DONE]"], "", None, None), ], ) def test_streaming( @@ -672,6 +668,52 @@ def test_streaming( assert response.output_metrics.text_words == len(expected_text.split()) assert response.output_metrics.text_characters == len(expected_text) + @pytest.mark.regression + @pytest.mark.parametrize( + "response", + [ + {"choices": [{"text": ""}], "usage": {}}, + {"choices": [], "usage": {}}, + {}, + ], + ) + def test_non_streaming_raises_for_unusable_terminal_payload( + self, valid_instances, generation_request, response + ): + """Test unusable non-streaming text response raises. + + ### WRITTEN BY AI ### + """ + instance = valid_instances + arguments = instance.format(generation_request) + compiled = instance.compile_non_streaming( + generation_request, arguments, response + ) + + with pytest.raises(ValueError, match="UNUSABLE_BACKEND_RESPONSE"): + instance.post_validation(compiled) + + @pytest.mark.regression + def test_streaming_raises_for_unusable_terminal_payload( + self, valid_instances, generation_request + ): + """Test unusable streaming text response raises. + + ### WRITTEN BY AI ### + """ + instance = valid_instances + arguments = instance.format(generation_request) + + for line in ["", "data: [DONE]"]: + result = instance.add_streaming_line(line) + if result is None: + break + + compiled = instance.compile_streaming(generation_request, arguments) + + with pytest.raises(ValueError, match="UNUSABLE_BACKEND_RESPONSE"): + instance.post_validation(compiled) + @pytest.mark.smoke @pytest.mark.parametrize( ("line", "expected_output"), @@ -1112,18 +1154,6 @@ def test_format_multimodal(self, valid_instances): 10, 5, ), - ( - {"choices": [{"message": {"content": ""}}], "usage": {}}, - "", - None, - None, - ), - ( - {"choices": [], "usage": {}}, - "", - None, - None, - ), ], ) def test_non_streaming( @@ -1177,12 +1207,6 @@ def test_non_streaming( None, None, ), - ( - ["", "data: [DONE]"], - "", - None, - None, - ), ], ) def test_streaming( @@ -1214,6 +1238,86 @@ def test_streaming( assert response.input_metrics.text_tokens == expected_input_tokens assert response.output_metrics.text_tokens == expected_output_tokens + @pytest.mark.regression + @pytest.mark.parametrize( + "response", + [ + {"choices": [{"message": {"content": ""}}], "usage": {}}, + {"choices": [], "usage": {}}, + ], + ) + def test_non_streaming_raises_for_unusable_terminal_payload( + self, valid_instances, generation_request, response + ): + """Test unusable non-streaming chat response raises. + + ### WRITTEN BY AI ### + """ + instance = valid_instances + arguments = instance.format(generation_request) + compiled = instance.compile_non_streaming( + generation_request, arguments, response + ) + + with pytest.raises(ValueError, match="UNUSABLE_BACKEND_RESPONSE"): + instance.post_validation(compiled) + + @pytest.mark.regression + def test_streaming_raises_for_unusable_terminal_payload( + self, valid_instances, generation_request + ): + """Test unusable streaming chat response raises. + + ### WRITTEN BY AI ### + """ + instance = valid_instances + arguments = instance.format(generation_request) + + for line in ["", "data: [DONE]"]: + result = instance.add_streaming_line(line) + if result is None: + break + + compiled = instance.compile_streaming(generation_request, arguments) + + with pytest.raises(ValueError, match="UNUSABLE_BACKEND_RESPONSE"): + instance.post_validation(compiled) + + @pytest.mark.regression + def test_non_streaming_tool_call_only_passes_validation( + self, valid_instances, generation_request + ): + """Tool-call-only response (no text) is valid and must not raise. + + ### WRITTEN BY AI ### + """ + instance = valid_instances + arguments = instance.format(generation_request) + response = { + "choices": [ + { + "message": { + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city":"NYC"}', + }, + } + ], + } + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5}, + } + result = instance.compile_non_streaming(generation_request, arguments, response) + instance.post_validation(result) + assert result.tool_calls is not None + assert len(result.tool_calls) == 1 + @pytest.mark.sanity def test_streaming_reasoning_tokens(self, valid_instances, generation_request): """Test that reasoning tokens are properly detected for TTFT measurement. @@ -1228,15 +1332,12 @@ def test_streaming_reasoning_tokens(self, valid_instances, generation_request): arguments = instance.format(generation_request) lines = [ - # First chunk has reasoning token ( 'data: {"id": "chatcmpl-123", "choices": ' '[{"index": 0, "delta": {"reasoning": "Okay"}}], "usage": {}}' ), - # More reasoning tokens 'data: {"choices": [{"delta": {"reasoning": ", let me"}}], "usage": {}}', 'data: {"choices": [{"delta": {"reasoning": " think..."}}], "usage": {}}', - # Finally content tokens 'data: {"choices": [{"delta": {"content": "Hello"}}], "usage": {}}', ( 'data: {"choices": [{"delta": {"content": " world!"}}], ' @@ -1256,17 +1357,13 @@ def test_streaming_reasoning_tokens(self, valid_instances, generation_request): elif result is None: break - # Verify that the first update happened on the first reasoning token (line 0) assert first_update_on_line == 0, ( f"Expected first token detection on line 0 (reasoning token), " f"but got {first_update_on_line}" ) - - # Verify all chunks with content were counted (5 lines with tokens) assert updated_count == 5 response = instance.compile_streaming(generation_request, arguments) - # Reasoning tokens should NOT appear in response.text; only content does assert "Okay" not in response.text assert "let me think..." not in response.text assert response.text == "Hello world!" @@ -1289,7 +1386,6 @@ def test_streaming_both_reasoning_and_content_in_same_chunk( arguments = instance.format(generation_request) lines = [ - # Chunk with both reasoning and content (edge case) ( 'data: {"choices": [{"delta": ' '{"reasoning": "Let me think...", "content": "Answer: "}}], ' @@ -1307,12 +1403,9 @@ def test_streaming_both_reasoning_and_content_in_same_chunk( if result > 0: updated_count += 1 - # First chunk has both reasoning and content (counts as 1 iteration) - # Second chunk has content only (counts as 1 iteration) assert updated_count == 2 response = instance.compile_streaming(generation_request, arguments) - # Reasoning text should NOT appear; only content is captured assert "Let me think..." not in response.text assert response.text == "Answer: 42" @@ -3002,18 +3095,6 @@ def test_format_input_items_image(self, valid_instances): 10, 8, ), - ( - {"id": "resp_789", "output": [], "usage": {}}, - "", - None, - None, - ), - ( - {"output": []}, - "", - None, - None, - ), ], ) def test_non_streaming( @@ -3107,29 +3188,6 @@ def test_non_streaming( None, None, ), - ( - [ - "event: response.created", - ( - "data: {" - '"type":"response.created",' - '"response":{"id":"resp_3"},' - '"sequence_number":0}' - ), - "", - "event: response.completed", - ( - "data: {" - '"type":"response.completed",' - '"response":{"id":"resp_3","usage":{}},' - '"sequence_number":2}' - ), - "data: [DONE]", - ], - "", - None, - None, - ), ], ) def test_streaming( @@ -3289,20 +3347,6 @@ def test_extract_line_data(self, valid_instances, line, expected_output): 10, 2, ), - ( - [ - "event: response.failed", - ( - "data: {" - '"type":"response.failed",' - '"response":{"id":"resp_fail_no_usage"},' - '"sequence_number":1}' - ), - ], - "", - None, - None, - ), ], ) def test_streaming_terminal_events( @@ -3332,6 +3376,57 @@ def test_streaming_terminal_events( assert response.input_metrics.text_tokens == expected_input_tokens assert response.output_metrics.text_tokens == expected_output_tokens + @pytest.mark.regression + @pytest.mark.parametrize( + "response", + [ + {"output": [], "usage": {}}, + {"output": [{"type": "message", "content": []}], "usage": {}}, + ], + ) + def test_non_streaming_raises_for_unusable_terminal_payload( + self, valid_instances, generation_request, response + ): + """Responses API empty output raises UNUSABLE_BACKEND_RESPONSE. + + ### WRITTEN BY AI ### + """ + instance = valid_instances + arguments = instance.format(generation_request) + + compiled = instance.compile_non_streaming( + generation_request, arguments, response + ) + + with pytest.raises(ValueError, match="UNUSABLE_BACKEND_RESPONSE"): + instance.post_validation(compiled) + + @pytest.mark.regression + def test_non_streaming_tool_call_only_passes_validation( + self, valid_instances, generation_request + ): + """Tool-call-only Responses API output is valid and must not raise. + + ### WRITTEN BY AI ### + """ + instance = valid_instances + arguments = instance.format(generation_request) + response = { + "output": [ + { + "type": "function_call", + "call_id": "call_1", + "name": "get_weather", + "arguments": '{"city":"NYC"}', + } + ], + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + result = instance.compile_non_streaming(generation_request, arguments, response) + instance.post_validation(result) + assert result.tool_calls is not None + assert len(result.tool_calls) == 1 + @pytest.mark.sanity def test_streaming_reasoning_triggers_ttft_not_content( self, valid_instances, generation_request