diff --git a/docs/guides/backends.md b/docs/guides/backends.md index 4f9d79b98..f810420c1 100644 --- a/docs/guides/backends.md +++ b/docs/guides/backends.md @@ -129,11 +129,34 @@ guidellm run \ This will include `temperature`, `top_p`, and `top_k` in every request body sent to the server. +## Structured Chat Content Payloads + +Some chat templates require metadata alongside the text in each structured content object. Pass these fields through `extras.content` in the `openai_http` backend configuration. GuideLLM adds them to every generated text content object for Chat Completions and Responses API requests. + +```bash +guidellm run \ + --backend '{ + "kind": "openai_http", + "target": "http://localhost:8000", + "model": "google/translategemma-12b-it", + "request_format": "/v1/chat/completions", + "extras": { + "content": { + "source_lang_code": "en", + "target_lang_code": "es" + } + } + }' \ + --data kind=synthetic_text,prompt_tokens=1000,output_tokens=1000 \ + --constraint kind=max_duration,seconds=60 +``` + ### How It Works The `--backend` config is parsed into keyword arguments for the backend constructor. The `extras` field within that config maps to a `GenerationRequestArguments` object that supports the following sub-fields: - `body`: A dictionary of key-value pairs merged into the HTTP request body. Use this for sampling parameters like `temperature`, `top_p`, `top_k`, `repetition_penalty`, etc. +- `content`: A dictionary of fields merged into each generated text content object. - `headers`: A dictionary of additional HTTP headers to include in requests. - `params`: A dictionary of query parameters to append to the request URL. diff --git a/src/guidellm/backends/openai/request_handlers.py b/src/guidellm/backends/openai/request_handlers.py index 1a0451682..ed9ba11bf 100644 --- a/src/guidellm/backends/openai/request_handlers.py +++ b/src/guidellm/backends/openai/request_handlers.py @@ -707,7 +707,10 @@ def _ensure_tool_format(tool: dict[str, Any]) -> dict[str, Any]: return tool def _format_prompts( - self, column_data: list[dict[str, Any]], column_type: str + self, + column_data: list, + column_type: str, + content_extras: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: """ Helper method to format different types of data columns @@ -716,7 +719,10 @@ def _format_prompts( formatted_data = [] for item in column_data: if column_type == "text_column": - formatted_data.append({"type": "text", "text": item}) + content = {"type": "text", "text": item} + if content_extras: + content.update(content_extras) + formatted_data.append(content) elif column_type == "image_column": formatted_data.append( { @@ -910,8 +916,14 @@ def _build_turn_messages( # noqa: C901 if prefix: messages.append({"role": "system", "content": prefix}) + extras = kwargs.get("extras") + content_extras = extras.content if extras is not None else None prompts = [ - self._format_prompts(req.columns.get(col, []), col) + self._format_prompts( + req.columns.get(col, []), + col, + content_extras, + ) for col in ( "text_column", "image_column", @@ -1018,8 +1030,14 @@ def format( # noqa: C901, PLR0912, PLR0915 if prefix: arguments.body["messages"].append({"role": "system", "content": prefix}) + extras = kwargs.get("extras") + content_extras = extras.content if extras is not None else None prompts = [ - self._format_prompts(data.columns.get(col, []), col) + self._format_prompts( + data.columns.get(col, []), + col, + content_extras, + ) for col in ( "text_column", "image_column", @@ -1497,12 +1515,18 @@ def _ensure_tool_format(tool: dict[str, Any]) -> dict[str, Any]: return tool def _format_prompts( - self, column_data: list, column_type: str + self, + column_data: list, + column_type: str, + content_extras: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: formatted_data: list[dict[str, Any]] = [] for item in column_data: if column_type == "text_column": - formatted_data.append({"type": "input_text", "text": item}) + content = {"type": "input_text", "text": item} + if content_extras: + content.update(content_extras) + formatted_data.append(content) elif column_type == "image_column": formatted_data.append( { @@ -1582,8 +1606,14 @@ def _build_turn_input_items( # noqa: C901 items.append({"role": "assistant", "content": content}) else: # Standard or tool_call turn: user content. + extras = kwargs.get("extras") + content_extras = extras.content if extras is not None else None prompts = [ - self._format_prompts(req.columns.get(col, []), col) + self._format_prompts( + req.columns.get(col, []), + col, + content_extras, + ) for col in ( "text_column", "image_column", @@ -1751,8 +1781,14 @@ def format( # noqa: C901 ) elif data.turn_type != "tool_response_injection": # Standard or tool_call turn: user content. + extras = kwargs.get("extras") + content_extras = extras.content if extras is not None else None prompts = [ - self._format_prompts(data.columns.get(col, []), col) + self._format_prompts( + data.columns.get(col, []), + col, + content_extras, + ) for col in ( "text_column", "image_column", diff --git a/src/guidellm/schemas/request.py b/src/guidellm/schemas/request.py index 1367395ce..370291b3f 100644 --- a/src/guidellm/schemas/request.py +++ b/src/guidellm/schemas/request.py @@ -72,6 +72,13 @@ class GenerationRequestArguments(StandardBaseDict): default=None, description="Files to include in the request, if applicable.", ) + content: dict[str, Any] | None = Field( + default=None, + description=( + "Additional fields to include in generated text content objects, " + "if applicable." + ), + ) def model_combine( self, additional: GenerationRequestArguments | dict[str, Any] diff --git a/tests/unit/backends/openai/test_http.py b/tests/unit/backends/openai/test_http.py index d053699ce..fc8a0f057 100644 --- a/tests/unit/backends/openai/test_http.py +++ b/tests/unit/backends/openai/test_http.py @@ -179,6 +179,34 @@ def test_server_history_with_responses_api(self): ) assert backend._args.server_history is True + @pytest.mark.asyncio + @pytest.mark.regression + async def test_content_extras_are_forwarded_to_request_handler( + self, + mock_request_handler, + ): + """The HTTP backend forwards content extras through its handler boundary. + + ## WRITTEN BY AI ## + """ + extras = GenerationRequestArguments( + content={ + "metadata": {"category": "support"}, + "priority": 1, + } + ) + backend = _make_backend( + target="http://localhost:8000", + model="test-model", + extras=extras, + ) + mock_handler, handler_patch = mock_request_handler + + with handler_patch: + await backend._prepare_resolve_request(GenerationRequest()) + + assert mock_handler.format.call_args.kwargs["extras"] == extras + @pytest.mark.smoke def test_factory_registration(self): """Test that OpenAIHTTPBackend is registered with Backend factory.""" diff --git a/tests/unit/backends/openai/test_request_handlers.py b/tests/unit/backends/openai/test_request_handlers.py index 287b22413..863efb5e8 100644 --- a/tests/unit/backends/openai/test_request_handlers.py +++ b/tests/unit/backends/openai/test_request_handlers.py @@ -29,7 +29,12 @@ GenerativeRequestFinalizer, GenerativeRequestFinalizerArgs, ) -from guidellm.schemas import GenerationRequest, GenerationResponse, UsageMetrics +from guidellm.schemas import ( + GenerationRequest, + GenerationRequestArguments, + GenerationResponse, + UsageMetrics, +) from guidellm.schemas.tool_call import ToolCall, ToolCallFunction from guidellm.settings import settings from guidellm.utils.registry import RegistryMixin @@ -502,7 +507,7 @@ def test_format_extras(self, valid_instances): """ instance = valid_instances data = GenerationRequest() - extras = {"body": {"temperature": 0.7, "top_p": 0.9}} + extras = GenerationRequestArguments(body={"temperature": 0.7, "top_p": 0.9}) result = instance.format(data, extras=extras) @@ -925,7 +930,7 @@ def test_format_extras(self, valid_instances): """ instance = valid_instances data = GenerationRequest() - extras = {"body": {"temperature": 0.5, "top_k": 40}} + extras = GenerationRequestArguments(body={"temperature": 0.5, "top_k": 40}) result = instance.format(data, extras=extras) @@ -953,6 +958,35 @@ def test_format_messages_text(self, valid_instances): assert result.body["messages"][0]["content"][1]["type"] == "text" assert result.body["messages"][0]["content"][1]["text"] == "How are you?" + @pytest.mark.regression + def test_content_extras_enrich_plain_text_only(self, valid_instances): + """Content extras enrich text without changing multimodal parts. + + ## WRITTEN BY AI ## + """ + data = GenerationRequest( + columns={ + "text_column": ["Describe this"], + "image_column": [{"image": "https://example.com/image.jpg"}], + } + ) + + result = valid_instances.format( + data, + extras=GenerationRequestArguments( + content={ + "metadata": {"category": "vision"}, + "priority": 1, + } + ), + ) + + text_content, image_content = result.body["messages"][0]["content"] + assert text_content["metadata"] == {"category": "vision"} + assert text_content["priority"] == 1 + assert "metadata" not in image_content + assert "priority" not in image_content + @pytest.mark.sanity def test_format_messages_prefix(self, valid_instances): """Test format method with prefix as system message. @@ -2061,7 +2095,10 @@ def test_format_strips_tool_choice_without_tools(self, valid_instances): turn_type="standard", ) - result = instance.format(data, extras={"body": {"tool_choice": "required"}}) + result = instance.format( + data, + extras=GenerationRequestArguments(body={"tool_choice": "required"}), + ) assert "tool_choice" not in result.body assert "tools" not in result.body @@ -2273,7 +2310,7 @@ def test_format_extras(self, valid_instances): ] }, ) - extras = {"body": {"language": "en", "temperature": 0.0}} + extras = GenerationRequestArguments(body={"language": "en", "temperature": 0.0}) result = instance.format(data, extras=extras) @@ -2626,6 +2663,30 @@ def test_chat_format_with_single_turn_history(self, valid_instances): assert messages[1]["content"] == "The answer is 4" assert messages[2]["role"] == "user" + @pytest.mark.regression + def test_content_extras_apply_to_history_and_current_turn(self, valid_instances): + """Content extras are applied consistently across conversation turns. + + ## WRITTEN BY AI ## + """ + prev_request = GenerationRequest(columns={"text_column": ["Previous"]}) + prev_response = GenerationResponse( + request_id="prev", + request_args=None, + text="Previous response", + ) + data = GenerationRequest(columns={"text_column": ["Current"]}) + + result = valid_instances.format( + data, + history=[(prev_request, prev_response)], + extras=GenerationRequestArguments(content={"priority": 3}), + ) + + messages = result.body["messages"] + assert messages[0]["content"][0]["priority"] == 3 + assert messages[2]["content"][0]["priority"] == 3 + @pytest.mark.sanity def test_chat_format_with_multi_turn_history(self, valid_instances): """Test format with multiple turns alternates user/assistant. @@ -2813,6 +2874,56 @@ def test_factory_registration(self): handler = OpenAIRequestHandlerFactory.create("/v1/responses") assert isinstance(handler, ResponsesRequestHandler) + @pytest.mark.regression + def test_content_extras_enrich_text(self, valid_instances): + """Content extras are added to Responses API text content. + + ## WRITTEN BY AI ## + """ + data = GenerationRequest(columns={"text_column": ["Handle this request"]}) + + result = valid_instances.format( + data, + extras=GenerationRequestArguments( + content={ + "priority": 2, + "metadata": {"category": "support"}, + } + ), + ) + + content = result.body["input"][0]["content"][0] + assert content == { + "type": "input_text", + "text": "Handle this request", + "priority": 2, + "metadata": {"category": "support"}, + } + + @pytest.mark.regression + def test_content_extras_apply_to_history_and_current_turn(self, valid_instances): + """Content extras apply to all Responses API conversation turns. + + ## WRITTEN BY AI ## + """ + prev_request = GenerationRequest(columns={"text_column": ["Previous"]}) + prev_response = GenerationResponse( + request_id="prev", + request_args=None, + text="Previous response", + ) + data = GenerationRequest(columns={"text_column": ["Current"]}) + + result = valid_instances.format( + data, + history=[(prev_request, prev_response)], + extras=GenerationRequestArguments(content={"priority": 3}), + ) + + input_items = result.body["input"] + assert input_items[0]["content"][0]["priority"] == 3 + assert input_items[2]["content"][0]["priority"] == 3 + @pytest.mark.smoke def test_format_minimal(self, valid_instances): """ @@ -4347,7 +4458,10 @@ def test_format_tool_choice_none_on_non_tool_turn(self, valid_instances): tools = [{"type": "function", "function": {"name": "fn", "parameters": {}}}] data = GenerationRequest(turn_type="standard") - result = instance.format(data, extras={"body": {"tools": tools}}) + result = instance.format( + data, + extras=GenerationRequestArguments(body={"tools": tools}), + ) assert result.body["tools"] == tools assert result.body["tool_choice"] == "none" @@ -4366,7 +4480,10 @@ def test_format_strips_tool_choice_without_tools(self, valid_instances): turn_type="standard", ) - result = instance.format(data, extras={"body": {"tool_choice": "required"}}) + result = instance.format( + data, + extras=GenerationRequestArguments(body={"tool_choice": "required"}), + ) assert "tool_choice" not in result.body assert "tools" not in result.body @@ -4644,7 +4761,7 @@ def test_format_extras(self, valid_instances): """ instance = valid_instances data = GenerationRequest() - extras = {"body": {"temperature": 0.5, "top_k": 40}} + extras = GenerationRequestArguments(body={"temperature": 0.5, "top_k": 40}) result = instance.format(data, extras=extras) @@ -4868,7 +4985,7 @@ def test_format_with_extras(self, valid_instances): """ instance = valid_instances data = GenerationRequest() - extras = {"body": {"user": "test-user"}} + extras = GenerationRequestArguments(body={"user": "test-user"}) result = instance.format(data, extras=extras) @@ -4973,7 +5090,7 @@ def test_tool_choice_none_when_expects_false(self, handler): }, turn_type="standard", ) - extras = {"body": {"tool_choice": "required"}} + extras = GenerationRequestArguments(body={"tool_choice": "required"}) result = handler.format(data, extras=extras) assert result.body["tool_choice"] == "none" @@ -4992,7 +5109,7 @@ def test_tool_choice_preserved_when_expects_true(self, handler): }, turn_type="client_tool_call", ) - extras = {"body": {"tool_choice": "required"}} + extras = GenerationRequestArguments(body={"tool_choice": "required"}) result = handler.format(data, extras=extras) assert result.body["tool_choice"] == "required" @@ -5011,7 +5128,7 @@ def test_auto_tool_choice_preserved_when_expects_true(self, handler): }, turn_type="client_tool_call", ) - extras = {"body": {"tool_choice": "auto"}} + extras = GenerationRequestArguments(body={"tool_choice": "auto"}) result = handler.format(data, extras=extras) assert result.body["tool_choice"] == "auto" diff --git a/tests/unit/schemas/test_request.py b/tests/unit/schemas/test_request.py index 423dde139..2105bfbf6 100644 --- a/tests/unit/schemas/test_request.py +++ b/tests/unit/schemas/test_request.py @@ -37,6 +37,7 @@ class TestGenerationRequestArguments: "params": {"limit": 10}, "body": {"prompt": "hello"}, "files": {"file": "data.txt"}, + "content": {"metadata": {"category": "support"}}, }, ], ids=["empty", "method_body", "method_headers_params", "all_fields"], @@ -57,7 +58,15 @@ def test_class_signatures(self): # Check fields fields = GenerationRequestArguments.model_fields - expected_fields = ["method", "stream", "headers", "params", "body", "files"] + expected_fields = [ + "method", + "stream", + "headers", + "params", + "body", + "files", + "content", + ] for field in expected_fields: assert field in fields @@ -72,7 +81,15 @@ def test_initialization(self, valid_instances): assert getattr(instance, key) == expected_value # Check defaults for fields not provided - for field in ["method", "stream", "headers", "params", "body", "files"]: + for field in [ + "method", + "stream", + "headers", + "params", + "body", + "files", + "content", + ]: if field not in constructor_args: assert getattr(instance, field) is None @@ -99,6 +116,10 @@ def test_invalid_initialization_values(self): with pytest.raises(ValidationError): GenerationRequestArguments(body="not_dict") + # Invalid content type + with pytest.raises(ValidationError): + GenerationRequestArguments(content="not_dict") + @pytest.mark.sanity def test_invalid_initialization_missing(self): """Test GenerationRequestArguments initialization without any fields.""" @@ -111,6 +132,7 @@ def test_invalid_initialization_missing(self): assert instance.params is None assert instance.body is None assert instance.files is None + assert instance.content is None @pytest.mark.smoke @pytest.mark.parametrize(