From 7fabc22c84a953ff5187af27a90d2f8e379ffc13 Mon Sep 17 00:00:00 2001 From: prasanna Date: Thu, 23 Jul 2026 00:44:32 +0530 Subject: [PATCH 1/4] feat(openai): support structured chat content metadata for ChatCompletions (#940) Models like `google/translategemma-12b-it` require structured text content objects with extra metadata fields (e.g., `source_lang_code` and `target_lang_code`) in `/v1/chat/completions` requests. Without these fields, server-side Jinja chat templates fail with HTTP 400 errors. This change adds support for appending custom metadata fields into structured chat content objects: - Added `content_extra_fields` to `OpenAIHTTPBackendArgs` and forwarded it to `ChatCompletionsRequestHandler._format_prompts()`. - Added the generic `--append-payloads` CLI option to pass key-value metadata. - Enabled preservation of dictionary prompt objects from custom dataset rows in `text_column`. - Updated `GenerativeRequestFinalizer.finalize_turn()` to safely extract string content from dictionary prompts when computing character and word usage metrics. - Added comprehensive unit test coverage under `tests/unit/backends/openai/` and `tests/unit/data/`. Resolves #940 Signed-off-by: prasanna --- docs/guides/backends.md | 33 ++++ src/guidellm/backends/openai/http.py | 34 ++++ .../backends/openai/request_handlers.py | 55 ++++++- src/guidellm/cli/run.py | 32 ++++ src/guidellm/data/finalizers/generative.py | 24 ++- tests/unit/backends/openai/test_http.py | 70 ++++++++ .../backends/openai/test_request_handlers.py | 142 +++++++++++++++++ tests/unit/cli/test_run.py | 58 +++++++ tests/unit/data/preprocessors/test_mappers.py | 150 ++++++++++++++++++ tests/unit/data/test_finalizers.py | 35 ++++ 10 files changed, 626 insertions(+), 7 deletions(-) create mode 100644 tests/unit/data/preprocessors/test_mappers.py diff --git a/docs/guides/backends.md b/docs/guides/backends.md index 4f9d79b98..70e5a2d89 100644 --- a/docs/guides/backends.md +++ b/docs/guides/backends.md @@ -129,6 +129,39 @@ guidellm run \ This will include `temperature`, `top_p`, and `top_k` in every request body sent to the server. +## Structured Chat Content Payloads + +Some chat templates require metadata alongside the text in each structured content object. GuideLLM supports this metadata from custom datasets and from the `--append-payloads` option for the `openai_http` chat-completions backend. + +### Per-prompt metadata from a custom dataset + +Store the structured text content object in the dataset's `prompt` field. Extra fields are preserved when GuideLLM builds the chat request: + +```json +{"prompt":{"type":"text","text":"The quick brown fox jumps over the lazy dog.","source_lang_code":"en","target_lang_code":"es"},"output_tokens_count":1000} +``` + +The default generative column mapper already maps `prompt` to `text_column`, so no custom column mapping is required: + +```bash +guidellm run \ + --backend 'kind=openai_http,target=http://localhost:8000,model=google/translategemma-12b-it,request_format=/v1/chat/completions' \ + --data '{"kind":"json_file","path":"translations.jsonl","load_kwargs":{"split":"train"}}' \ + --constraint kind=max_requests,count=10 +``` + +### Applying or overriding metadata from the CLI + +Use `--append-payloads` when every request should receive the same fields. CLI values override matching metadata in a structured dataset prompt. The generated `type` and `text` fields are reserved and cannot be overridden. + +```bash +guidellm run \ + --backend 'kind=openai_http,target=http://localhost:8000,model=google/translategemma-12b-it,request_format=/v1/chat/completions' \ + --append-payloads '{"source_lang_code":"en","target_lang_code":"es"}' \ + --data kind=synthetic_text,prompt_tokens=1000,output_tokens=1000 \ + --constraint kind=max_duration,seconds=60 +``` + ### How It Works The `--backend` config is parsed into keyword arguments for the backend constructor. The `extras` field within that config maps to a `GenerationRequestArguments` object that supports the following sub-fields: diff --git a/src/guidellm/backends/openai/http.py b/src/guidellm/backends/openai/http.py index f6ec2beab..e79d813e7 100644 --- a/src/guidellm/backends/openai/http.py +++ b/src/guidellm/backends/openai/http.py @@ -139,6 +139,21 @@ class OpenAIHTTPBackendArgs(BackendArgs): default=None, description="Additional parameters to include in generation requests.", ) + append_payloads: dict[str, Any] | None = Field( + default=None, + description=( + "Additional fields to append to every text content object in chat " + "completion requests. These fields override matching metadata from " + "structured dataset prompts. The reserved fields 'type' and 'text' " + "cannot be overridden." + ), + examples=[ + { + "metadata": {"category": "support"}, + "priority": 1, + } + ], + ) max_tokens: int | None = Field( default=None, validation_alias=AliasChoices("max_tokens", "max_completion_tokens"), @@ -208,6 +223,24 @@ def validate_server_history(self): ) return self + @model_validator(mode="after") + def validate_append_payloads(self): + """Validate append payloads are safe and scoped to chat completions.""" + if not self.append_payloads: + return self + if self.request_format != "/v1/chat/completions": + raise ValueError( + "append_payloads is only supported with the " + "'/v1/chat/completions' request format" + ) + reserved_fields = {"type", "text"}.intersection(self.append_payloads) + if reserved_fields: + fields = ", ".join(sorted(reserved_fields)) + raise ValueError( + f"append_payloads cannot override reserved content fields: {fields}" + ) + return self + @Backend.register("openai_http") class OpenAIHTTPBackend(Backend): @@ -428,6 +461,7 @@ async def _prepare_resolve_request( max_tokens=self._args.max_tokens, server_history=self._args.server_history, multiturn_reasoning=self._args.multiturn_reasoning, + append_payloads=self._args.append_payloads, ) request_url = f"{self._args.target}/{request_path}" diff --git a/src/guidellm/backends/openai/request_handlers.py b/src/guidellm/backends/openai/request_handlers.py index 1a0451682..08822529f 100644 --- a/src/guidellm/backends/openai/request_handlers.py +++ b/src/guidellm/backends/openai/request_handlers.py @@ -706,17 +706,56 @@ def _ensure_tool_format(tool: dict[str, Any]) -> dict[str, Any]: return {"type": tool.get("type", "function"), "function": fn} return tool + @staticmethod + def _format_text_prompt( + item: Any, + append_payloads: dict[str, Any] | None, + ) -> dict[str, Any]: + """Format one plain or structured dataset prompt as text content. + + :param item: A plain string or structured text content dictionary. + :param append_payloads: Backend fields that override dataset metadata. + :return: A copied, validated text content dictionary. + :raises ValueError: If the prompt is not valid structured text content. + """ + if isinstance(item, str): + content = {"type": "text", "text": item} + elif isinstance(item, dict): + content = item.copy() + content.setdefault("type", "text") + if content["type"] != "text": + raise ValueError("Structured text prompts must use content type 'text'") + if not isinstance(content.get("text"), str): + raise ValueError( + "Structured text prompts must contain a string 'text' field" + ) + else: + raise ValueError( + "Text prompts must be strings or structured content objects" + ) + + if append_payloads: + content.update(append_payloads) + return content + def _format_prompts( - self, column_data: list[dict[str, Any]], column_type: str + self, + column_data: list[Any], + column_type: str, + append_payloads: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: """ Helper method to format different types of data columns into the appropriate structure for chat messages. + + Structured text dictionaries are copied as-is so model-specific metadata + from datasets is preserved. Backend-provided ``append_payloads`` are applied + last and therefore override matching dataset metadata. """ formatted_data = [] for item in column_data: if column_type == "text_column": - formatted_data.append({"type": "text", "text": item}) + formatted_data.append(self._format_text_prompt(item, append_payloads)) elif column_type == "image_column": formatted_data.append( { @@ -911,7 +950,11 @@ def _build_turn_messages( # noqa: C901 messages.append({"role": "system", "content": prefix}) prompts = [ - self._format_prompts(req.columns.get(col, []), col) + self._format_prompts( + req.columns.get(col, []), + col, + kwargs.get("append_payloads") if col == "text_column" else None, + ) for col in ( "text_column", "image_column", @@ -1019,7 +1062,11 @@ def format( # noqa: C901, PLR0912, PLR0915 arguments.body["messages"].append({"role": "system", "content": prefix}) prompts = [ - self._format_prompts(data.columns.get(col, []), col) + self._format_prompts( + data.columns.get(col, []), + col, + kwargs.get("append_payloads") if col == "text_column" else None, + ) for col in ( "text_column", "image_column", diff --git a/src/guidellm/cli/run.py b/src/guidellm/cli/run.py index a8eefcee7..8d8cd554e 100644 --- a/src/guidellm/cli/run.py +++ b/src/guidellm/cli/run.py @@ -2,6 +2,7 @@ import asyncio from pathlib import Path +from typing import Any import click from pydantic import ValidationError @@ -28,6 +29,22 @@ ] +def _parse_append_payloads( + ctx: click.Context, param: click.Parameter, value: str | None +) -> dict[str, Any] | None: + """Parse and validate structured content payload fields from the CLI.""" + if value is None: + return None + parsed = cli_tools.parse_arguments(ctx, param, value) + if not isinstance(parsed, dict): + raise click.BadParameter( + "must be a JSON, YAML, or key=value object", + ctx=ctx, + param=param, + ) + return parsed + + @click.command( "run", help=( @@ -68,6 +85,15 @@ ), ) @registry_options_from_model(model=BenchmarkArgs, group_key="spec") +@click.option( + "--append-payloads", + callback=_parse_append_payloads, + help=( + "Append key-value fields to structured content objects in chat completion " + "requests. Values override matching metadata from custom datasets. " + 'Example: `--append-payloads \'{"key":"value"}\'`' + ), +) @click.option( "--override", "benchmarks", @@ -106,6 +132,12 @@ def run(**kwargs): # noqa: C901, PLR0915 disable_console_interactive = ( kwargs.pop("disable_console_interactive", False) or disable_console ) + append_payloads = kwargs.pop("append_payloads", None) + if append_payloads is not None: + spec = kwargs.setdefault("spec", {}) + backend = spec.setdefault("backend", {}) + backend.setdefault("kind", "openai_http") + backend["append_payloads"] = append_payloads console = Console() if not disable_console else None if console: diff --git a/src/guidellm/data/finalizers/generative.py b/src/guidellm/data/finalizers/generative.py index 90e9d1905..53acebcc6 100644 --- a/src/guidellm/data/finalizers/generative.py +++ b/src/guidellm/data/finalizers/generative.py @@ -49,6 +49,24 @@ class GenerativeRequestFinalizer( def __init__(self, config: GenerativeRequestFinalizerArgs) -> None: self.config = config + @staticmethod + def _extract_prompt_text(prompt: Any) -> str: + """Extract metric-bearing text from a plain or structured prompt. + + :param prompt: A plain string or structured text content dictionary. + :return: The prompt text used for character and word metrics. + :raises ValueError: If the prompt does not contain valid text. + """ + if isinstance(prompt, str): + return prompt + if isinstance(prompt, dict) and isinstance(prompt.get("text"), str): + return prompt["text"] + if isinstance(prompt, dict): + raise ValueError( + "Structured text prompts must contain a string 'text' field" + ) + raise ValueError("Text prompts must be strings or structured content objects") + def __call__( self, items: list[dict[str, Any]] ) -> list[tuple[GenerationRequest, RequestSettings]]: @@ -106,11 +124,11 @@ def finalize_turn( # noqa: C901 PLR0912 input_metrics.add_text_metrics(prefix) # Count words in text prompts - for text in columns.get("text_column", []): - if not text: + for prompt in columns.get("text_column", []): + if not prompt: continue - input_metrics.add_text_metrics(text) + input_metrics.add_text_metrics(self._extract_prompt_text(prompt)) # Count pixels and bytes in images for image in columns.get("image_column", []): diff --git a/tests/unit/backends/openai/test_http.py b/tests/unit/backends/openai/test_http.py index d053699ce..abbf1c423 100644 --- a/tests/unit/backends/openai/test_http.py +++ b/tests/unit/backends/openai/test_http.py @@ -179,6 +179,76 @@ def test_server_history_with_responses_api(self): ) assert backend._args.server_history is True + @pytest.mark.sanity + def test_append_payloads_are_accepted_for_chat_completions(self): + """Chat backends accept structured content payload defaults. + + ## WRITTEN BY AI ## + """ + backend = _make_backend( + target="http://localhost:8000", + append_payloads={ + "metadata": {"category": "support"}, + "priority": 1, + }, + ) + + assert backend._args.append_payloads == { + "metadata": {"category": "support"}, + "priority": 1, + } + + @pytest.mark.sanity + @pytest.mark.parametrize("reserved_field", ["type", "text"]) + def test_append_payloads_reject_reserved_fields(self, reserved_field): + """Append payloads cannot replace GuideLLM-owned content fields. + + ## WRITTEN BY AI ## + """ + with pytest.raises(ValidationError, match="reserved content fields"): + _make_backend( + target="http://localhost:8000", + append_payloads={reserved_field: "replacement"}, + ) + + @pytest.mark.sanity + def test_append_payloads_reject_non_chat_endpoint(self): + """Append payloads are isolated from non-chat request handlers. + + ## WRITTEN BY AI ## + """ + with pytest.raises(ValidationError, match="only supported"): + _make_backend( + target="http://localhost:8000", + request_format="/v1/completions", + append_payloads={"priority": 1}, + ) + + @pytest.mark.asyncio + @pytest.mark.regression + async def test_append_payloads_are_forwarded_to_request_handler( + self, mock_request_handler + ): + """The HTTP backend forwards append payloads through its handler boundary. + + ## WRITTEN BY AI ## + """ + payloads = { + "metadata": {"category": "support"}, + "priority": 1, + } + backend = _make_backend( + target="http://localhost:8000", + model="test-model", + append_payloads=payloads, + ) + mock_handler, handler_patch = mock_request_handler + + with handler_patch: + await backend._prepare_resolve_request(GenerationRequest()) + + assert mock_handler.format.call_args.kwargs["append_payloads"] == payloads + @pytest.mark.smoke def test_factory_registration(self): """Test that OpenAIHTTPBackend is registered with Backend factory.""" diff --git a/tests/unit/backends/openai/test_request_handlers.py b/tests/unit/backends/openai/test_request_handlers.py index 287b22413..85924abaf 100644 --- a/tests/unit/backends/openai/test_request_handlers.py +++ b/tests/unit/backends/openai/test_request_handlers.py @@ -953,6 +953,102 @@ def test_format_messages_text(self, valid_instances): assert result.body["messages"][0]["content"][1]["type"] == "text" assert result.body["messages"][0]["content"][1]["text"] == "How are you?" + @pytest.mark.regression + def test_format_preserves_structured_text_metadata(self, valid_instances): + """Structured dataset prompts retain model-specific content fields. + + ## WRITTEN BY AI ## + """ + prompt = { + "type": "text", + "text": "Handle this request", + "metadata": {"category": "support"}, + "priority": 1, + } + data = GenerationRequest(columns={"text_column": [prompt]}) + + result = valid_instances.format(data) + + content = result.body["messages"][0]["content"][0] + assert content == prompt + assert content is not prompt + + @pytest.mark.regression + def test_append_payloads_override_structured_text_metadata(self, valid_instances): + """CLI payload values take precedence over custom dataset metadata. + + ## WRITTEN BY AI ## + """ + prompt = { + "type": "text", + "text": "Handle this request", + "priority": 1, + "metadata": {"source": "dataset"}, + "dataset_only": True, + } + data = GenerationRequest(columns={"text_column": [prompt]}) + + result = valid_instances.format( + data, + append_payloads={ + "priority": 2, + "metadata": {"source": "cli"}, + "cli_only": True, + }, + ) + + content = result.body["messages"][0]["content"][0] + assert content == { + "type": "text", + "text": "Handle this request", + "priority": 2, + "metadata": {"source": "cli"}, + "dataset_only": True, + "cli_only": True, + } + assert prompt["priority"] == 1 + assert prompt["metadata"] == {"source": "dataset"} + + @pytest.mark.regression + def test_append_payloads_enrich_plain_text_only(self, valid_instances): + """Append payloads enrich text without changing multimodal parts. + + ## WRITTEN BY AI ## + """ + data = GenerationRequest( + columns={ + "text_column": ["Describe this"], + "image_column": [{"image": "https://example.com/image.jpg"}], + } + ) + + result = valid_instances.format( + data, + append_payloads={ + "metadata": {"category": "vision"}, + "priority": 1, + }, + ) + + text_content, image_content = result.body["messages"][0]["content"] + assert text_content["metadata"] == {"category": "vision"} + assert text_content["priority"] == 1 + assert "metadata" not in image_content + assert "priority" not in image_content + + @pytest.mark.regression + def test_format_rejects_invalid_structured_text(self, valid_instances): + """Invalid structured prompt objects are rejected explicitly. + + ## WRITTEN BY AI ## + """ + data = GenerationRequest( + columns={"text_column": [{"type": "text", "text": 123}]} + ) + + with pytest.raises(ValueError, match="must contain a string 'text' field"): + valid_instances.format(data) + @pytest.mark.sanity def test_format_messages_prefix(self, valid_instances): """Test format method with prefix as system message. @@ -2626,6 +2722,52 @@ def test_chat_format_with_single_turn_history(self, valid_instances): assert messages[1]["content"] == "The answer is 4" assert messages[2]["role"] == "user" + @pytest.mark.regression + def test_append_payloads_override_history_and_current_metadata( + self, valid_instances + ): + """Append payload precedence is consistent across conversation turns. + + ## WRITTEN BY AI ## + """ + prev_request = GenerationRequest( + columns={ + "text_column": [ + { + "type": "text", + "text": "Previous", + "priority": 1, + } + ] + } + ) + prev_response = GenerationResponse( + request_id="prev", + request_args=None, + text="Previous response", + ) + data = GenerationRequest( + columns={ + "text_column": [ + { + "type": "text", + "text": "Current", + "priority": 2, + } + ] + } + ) + + result = valid_instances.format( + data, + history=[(prev_request, prev_response)], + append_payloads={"priority": 3}, + ) + + messages = result.body["messages"] + assert messages[0]["content"][0]["priority"] == 3 + assert messages[2]["content"][0]["priority"] == 3 + @pytest.mark.sanity def test_chat_format_with_multi_turn_history(self, valid_instances): """Test format with multiple turns alternates user/assistant. diff --git a/tests/unit/cli/test_run.py b/tests/unit/cli/test_run.py index 1842b9a02..36e82ad95 100644 --- a/tests/unit/cli/test_run.py +++ b/tests/unit/cli/test_run.py @@ -1,9 +1,12 @@ """Tests for ``guidellm run`` CLI error translation.""" +from unittest.mock import AsyncMock, patch + import pytest from click.testing import CliRunner from guidellm.__main__ import cli +from guidellm.backends.openai.http import OpenAIHTTPBackendArgs @pytest.mark.regression @@ -57,3 +60,58 @@ def test_run_allows_synthetic_text_without_output_tokens(): assert "Invalid value for --data" not in result.output assert "output_tokens" not in result.output + + +@pytest.mark.regression +def test_run_parses_append_payloads_into_openai_backend(): + """The top-level option is stored in the registered backend configuration. + + ## WRITTEN BY AI ## + """ + runner = CliRunner() + benchmark = AsyncMock() + + with patch("guidellm.cli.run.benchmark_generative_text", benchmark): + result = runner.invoke( + cli, + [ + "run", + "--backend", + "kind=openai_http,target=http://localhost:8000", + "--append-payloads", + '{"metadata":{"category":"support"},"priority":1}', + "--data", + "kind=synthetic_text,prompt_tokens=128", + "--disable-console", + ], + ) + + assert result.exit_code == 0, result.output + args = benchmark.await_args.kwargs["args"] + assert isinstance(args.spec.backend, OpenAIHTTPBackendArgs) + assert args.spec.backend.append_payloads == { + "metadata": {"category": "support"}, + "priority": 1, + } + + +@pytest.mark.regression +def test_run_rejects_non_object_append_payloads(): + """The CLI reports a clear error when append payloads is not an object. + + ## WRITTEN BY AI ## + """ + result = CliRunner().invoke( + cli, + [ + "run", + "--append-payloads", + '["not", "an", "object"]', + "--data", + "kind=synthetic_text,prompt_tokens=128", + ], + ) + + assert result.exit_code != 0 + assert "Invalid value for '--append-payloads'" in result.output + assert "must be a JSON, YAML, or key=value object" in result.output diff --git a/tests/unit/data/preprocessors/test_mappers.py b/tests/unit/data/preprocessors/test_mappers.py new file mode 100644 index 000000000..f75660420 --- /dev/null +++ b/tests/unit/data/preprocessors/test_mappers.py @@ -0,0 +1,150 @@ +"""Tests for generative dataset column mapping.""" + +from typing import Any + +import pytest +from datasets import Dataset + +from guidellm.data.preprocessors.mappers import ( + GenerativeColumnMapper, + GenerativeColumnMapperArgs, +) + + +def _structured_prompt( + text: str = "A structured prompt", + payload: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Build a structured text prompt for mapper tests.""" + return { + "type": "text", + "text": text, + **(payload or {}), + } + + +@pytest.mark.regression +def test_generative_mapper_preserves_structured_prompt_objects(): + """Custom dataset prompt dictionaries pass through the mapper unchanged. + + ## WRITTEN BY AI ## + """ + prompt = _structured_prompt() + dataset = Dataset.from_list([{"prompt": prompt}]) + mapper = GenerativeColumnMapper(GenerativeColumnMapperArgs()) + mapper.setup_data([dataset]) + + result = mapper([{"dataset": dataset[0]}]) + + assert result == [{"text_column": [prompt]}] + + +@pytest.mark.smoke +def test_generative_mapper_keeps_plain_text_behavior(): + """Plain prompts and token-count columns retain their existing mapping. + + ## WRITTEN BY AI ## + """ + dataset = Dataset.from_list([{"prompt": "A plain prompt", "input_tokens_count": 3}]) + mapper = GenerativeColumnMapper(GenerativeColumnMapperArgs()) + mapper.setup_data([dataset]) + + result = mapper([{"dataset": dataset[0]}]) + + assert result == [ + { + "prompt_tokens_count_column": [3], + "text_column": ["A plain prompt"], + } + ] + + +@pytest.mark.sanity +def test_generative_mapper_preserves_structured_prompt_with_explicit_mapping(): + """Structured prompts work when the source column has a custom name. + + ## WRITTEN BY AI ## + """ + prompt = _structured_prompt(payload={"request_context": "custom-column"}) + dataset = Dataset.from_list([{"content_payload": prompt}]) + mapper = GenerativeColumnMapper( + GenerativeColumnMapperArgs(column_mappings={"text_column": "content_payload"}) + ) + mapper.setup_data([dataset]) + + result = mapper([{"dataset": dataset[0]}]) + + assert result == [{"text_column": [prompt]}] + + +@pytest.mark.regression +def test_generative_mapper_preserves_nested_and_optional_metadata(): + """Nested, list, boolean, and null metadata survive column mapping. + + ## WRITTEN BY AI ## + """ + prompt = _structured_prompt( + payload={ + "metadata": { + "category": "support", + "labels": ["billing", "priority"], + }, + "enabled": True, + "optional_value": None, + } + ) + dataset = Dataset.from_list([{"prompt": prompt}]) + mapper = GenerativeColumnMapper(GenerativeColumnMapperArgs()) + mapper.setup_data([dataset]) + + result = mapper([{"dataset": dataset[0]}]) + + mapped_prompt = result[0]["text_column"][0] + assert mapped_prompt["metadata"] == { + "category": "support", + "labels": ["billing", "priority"], + } + assert mapped_prompt["enabled"] is True + assert mapped_prompt["optional_value"] is None + + +@pytest.mark.regression +def test_generative_mapper_keeps_per_row_payloads_independent(): + """Metadata from one dataset row does not leak into another. + + ## WRITTEN BY AI ## + """ + prompts = [ + _structured_prompt(text="First item", payload={"request_id": "one"}), + _structured_prompt(text="Second item", payload={"request_id": "two"}), + ] + dataset = Dataset.from_list([{"prompt": prompt} for prompt in prompts]) + mapper = GenerativeColumnMapper(GenerativeColumnMapperArgs()) + mapper.setup_data([dataset]) + + results = [mapper([{"dataset": dataset[index]}]) for index in range(2)] + + assert results[0][0]["text_column"][0] == prompts[0] + assert results[1][0]["text_column"][0] == prompts[1] + assert results[0][0]["text_column"][0]["request_id"] == "one" + assert results[1][0]["text_column"][0]["request_id"] == "two" + + +@pytest.mark.regression +def test_generative_mapper_preserves_structured_multiturn_prompts(): + """Turn-suffixed structured prompt columns map to separate turns. + + ## WRITTEN BY AI ## + """ + first_turn = _structured_prompt(text="First turn", payload={"turn_id": 0}) + second_turn = _structured_prompt(text="Second turn", payload={"turn_id": 1}) + dataset = Dataset.from_list([{"prompt_0": first_turn, "prompt_1": second_turn}]) + mapper = GenerativeColumnMapper(GenerativeColumnMapperArgs()) + mapper.setup_data([dataset]) + + result = mapper([{"dataset": dataset[0]}]) + + assert result == [ + {"text_column": [first_turn]}, + {"text_column": [second_turn]}, + ] diff --git a/tests/unit/data/test_finalizers.py b/tests/unit/data/test_finalizers.py index 9aa69ecfd..bb84fec23 100644 --- a/tests/unit/data/test_finalizers.py +++ b/tests/unit/data/test_finalizers.py @@ -147,6 +147,41 @@ def test_finalize_multi_value_text_columns(self, valid_instances): assert gen_req.input_metrics.text_words > 0 assert gen_req.input_metrics.text_characters > 0 + @pytest.mark.regression + def test_finalize_structured_text_preserves_metadata(self, valid_instances): + """Structured prompts retain metadata while metrics use their text. + + ## WRITTEN BY AI ## + """ + prompt = { + "type": "text", + "text": "The quick brown fox", + "metadata": {"category": "example"}, + "priority": 1, + } + + gen_req, _ = valid_instances.finalize_turn({"text_column": [prompt]}) + + assert gen_req.columns["text_column"] == [prompt] + assert gen_req.input_metrics.text_words == 4 + assert gen_req.input_metrics.text_characters == len(prompt["text"]) + + @pytest.mark.regression + def test_finalize_rejects_structured_text_without_string_text( + self, valid_instances + ): + """Malformed structured prompts fail with a useful validation message. + + ## WRITTEN BY AI ## + """ + with pytest.raises( + ValueError, + match="must contain a string 'text' field", + ): + valid_instances.finalize_turn( + {"text_column": [{"type": "text", "text": 123}]} + ) + @pytest.mark.sanity def test_finalize_multi_value_image_columns(self, valid_instances): """Test finalize sums image pixels and bytes across multiple images. From ccd9a479a5bf547c31c683eba555197432043681 Mon Sep 17 00:00:00 2001 From: prasanna Date: Mon, 27 Jul 2026 21:46:39 +0530 Subject: [PATCH 2/4] configure text content fields through extras Replace --append-payloads and structured dataset prompt metadata with backend extras.content. Apply the configured fields to generated text content for Chat Completions and Responses, including conversation history. Co-Authored-By: Prasanna <123716600+Prasannajaga@users.noreply.github.com> Signed-off-by: prasanna --- docs/guides/backends.md | 38 ++-- src/guidellm/backends/openai/http.py | 34 ---- .../backends/openai/request_handlers.py | 90 ++++----- src/guidellm/cli/run.py | 32 ---- src/guidellm/data/finalizers/generative.py | 24 +-- src/guidellm/schemas/request.py | 10 + tests/unit/backends/openai/test_http.py | 66 ++----- .../backends/openai/test_request_handlers.py | 172 +++++++----------- tests/unit/cli/test_run.py | 58 ------ tests/unit/data/preprocessors/test_mappers.py | 150 --------------- tests/unit/data/test_finalizers.py | 35 ---- tests/unit/schemas/test_request.py | 26 ++- 12 files changed, 177 insertions(+), 558 deletions(-) delete mode 100644 tests/unit/data/preprocessors/test_mappers.py diff --git a/docs/guides/backends.md b/docs/guides/backends.md index 70e5a2d89..f810420c1 100644 --- a/docs/guides/backends.md +++ b/docs/guides/backends.md @@ -131,33 +131,22 @@ This will include `temperature`, `top_p`, and `top_k` in every request body sent ## Structured Chat Content Payloads -Some chat templates require metadata alongside the text in each structured content object. GuideLLM supports this metadata from custom datasets and from the `--append-payloads` option for the `openai_http` chat-completions backend. - -### Per-prompt metadata from a custom dataset - -Store the structured text content object in the dataset's `prompt` field. Extra fields are preserved when GuideLLM builds the chat request: - -```json -{"prompt":{"type":"text","text":"The quick brown fox jumps over the lazy dog.","source_lang_code":"en","target_lang_code":"es"},"output_tokens_count":1000} -``` - -The default generative column mapper already maps `prompt` to `text_column`, so no custom column mapping is required: - -```bash -guidellm run \ - --backend 'kind=openai_http,target=http://localhost:8000,model=google/translategemma-12b-it,request_format=/v1/chat/completions' \ - --data '{"kind":"json_file","path":"translations.jsonl","load_kwargs":{"split":"train"}}' \ - --constraint kind=max_requests,count=10 -``` - -### Applying or overriding metadata from the CLI - -Use `--append-payloads` when every request should receive the same fields. CLI values override matching metadata in a structured dataset prompt. The generated `type` and `text` fields are reserved and cannot be overridden. +Some chat templates require metadata alongside the text in each structured content object. Pass these fields through `extras.content` in the `openai_http` backend configuration. GuideLLM adds them to every generated text content object for Chat Completions and Responses API requests. ```bash guidellm run \ - --backend 'kind=openai_http,target=http://localhost:8000,model=google/translategemma-12b-it,request_format=/v1/chat/completions' \ - --append-payloads '{"source_lang_code":"en","target_lang_code":"es"}' \ + --backend '{ + "kind": "openai_http", + "target": "http://localhost:8000", + "model": "google/translategemma-12b-it", + "request_format": "/v1/chat/completions", + "extras": { + "content": { + "source_lang_code": "en", + "target_lang_code": "es" + } + } + }' \ --data kind=synthetic_text,prompt_tokens=1000,output_tokens=1000 \ --constraint kind=max_duration,seconds=60 ``` @@ -167,6 +156,7 @@ guidellm run \ The `--backend` config is parsed into keyword arguments for the backend constructor. The `extras` field within that config maps to a `GenerationRequestArguments` object that supports the following sub-fields: - `body`: A dictionary of key-value pairs merged into the HTTP request body. Use this for sampling parameters like `temperature`, `top_p`, `top_k`, `repetition_penalty`, etc. +- `content`: A dictionary of fields merged into each generated text content object. - `headers`: A dictionary of additional HTTP headers to include in requests. - `params`: A dictionary of query parameters to append to the request URL. diff --git a/src/guidellm/backends/openai/http.py b/src/guidellm/backends/openai/http.py index e79d813e7..f6ec2beab 100644 --- a/src/guidellm/backends/openai/http.py +++ b/src/guidellm/backends/openai/http.py @@ -139,21 +139,6 @@ class OpenAIHTTPBackendArgs(BackendArgs): default=None, description="Additional parameters to include in generation requests.", ) - append_payloads: dict[str, Any] | None = Field( - default=None, - description=( - "Additional fields to append to every text content object in chat " - "completion requests. These fields override matching metadata from " - "structured dataset prompts. The reserved fields 'type' and 'text' " - "cannot be overridden." - ), - examples=[ - { - "metadata": {"category": "support"}, - "priority": 1, - } - ], - ) max_tokens: int | None = Field( default=None, validation_alias=AliasChoices("max_tokens", "max_completion_tokens"), @@ -223,24 +208,6 @@ def validate_server_history(self): ) return self - @model_validator(mode="after") - def validate_append_payloads(self): - """Validate append payloads are safe and scoped to chat completions.""" - if not self.append_payloads: - return self - if self.request_format != "/v1/chat/completions": - raise ValueError( - "append_payloads is only supported with the " - "'/v1/chat/completions' request format" - ) - reserved_fields = {"type", "text"}.intersection(self.append_payloads) - if reserved_fields: - fields = ", ".join(sorted(reserved_fields)) - raise ValueError( - f"append_payloads cannot override reserved content fields: {fields}" - ) - return self - @Backend.register("openai_http") class OpenAIHTTPBackend(Backend): @@ -461,7 +428,6 @@ async def _prepare_resolve_request( max_tokens=self._args.max_tokens, server_history=self._args.server_history, multiturn_reasoning=self._args.multiturn_reasoning, - append_payloads=self._args.append_payloads, ) request_url = f"{self._args.target}/{request_path}" diff --git a/src/guidellm/backends/openai/request_handlers.py b/src/guidellm/backends/openai/request_handlers.py index 08822529f..28d22837a 100644 --- a/src/guidellm/backends/openai/request_handlers.py +++ b/src/guidellm/backends/openai/request_handlers.py @@ -193,6 +193,21 @@ def _check_streaming_error(data: Any) -> None: raise ValueError(f"Streaming response returned an error: {message}") +def _get_content_extras( + extras: GenerationRequestArguments | dict[str, Any] | None, +) -> dict[str, Any] | None: + """Extract content-object fields from generation request extras. + + :param extras: Additional generation request arguments. + :return: Fields to merge into generated text content objects. + """ + if isinstance(extras, GenerationRequestArguments): + return extras.content + if isinstance(extras, dict): + return extras.get("content") + return None + + class WSEventResult(Enum): """Classification of a processed WebSocket streaming event.""" @@ -706,56 +721,23 @@ def _ensure_tool_format(tool: dict[str, Any]) -> dict[str, Any]: return {"type": tool.get("type", "function"), "function": fn} return tool - @staticmethod - def _format_text_prompt( - item: Any, - append_payloads: dict[str, Any] | None, - ) -> dict[str, Any]: - """Format one plain or structured dataset prompt as text content. - - :param item: A plain string or structured text content dictionary. - :param append_payloads: Backend fields that override dataset metadata. - :return: A copied, validated text content dictionary. - :raises ValueError: If the prompt is not valid structured text content. - """ - if isinstance(item, str): - content = {"type": "text", "text": item} - elif isinstance(item, dict): - content = item.copy() - content.setdefault("type", "text") - if content["type"] != "text": - raise ValueError("Structured text prompts must use content type 'text'") - if not isinstance(content.get("text"), str): - raise ValueError( - "Structured text prompts must contain a string 'text' field" - ) - else: - raise ValueError( - "Text prompts must be strings or structured content objects" - ) - - if append_payloads: - content.update(append_payloads) - return content - def _format_prompts( self, - column_data: list[Any], + column_data: list, column_type: str, - append_payloads: dict[str, Any] | None = None, + content_extras: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: """ Helper method to format different types of data columns into the appropriate structure for chat messages. - - Structured text dictionaries are copied as-is so model-specific metadata - from datasets is preserved. Backend-provided ``append_payloads`` are applied - last and therefore override matching dataset metadata. """ formatted_data = [] for item in column_data: if column_type == "text_column": - formatted_data.append(self._format_text_prompt(item, append_payloads)) + content = {"type": "text", "text": item} + if content_extras: + content.update(content_extras) + formatted_data.append(content) elif column_type == "image_column": formatted_data.append( { @@ -949,11 +931,12 @@ def _build_turn_messages( # noqa: C901 if prefix: messages.append({"role": "system", "content": prefix}) + content_extras = _get_content_extras(kwargs.get("extras")) prompts = [ self._format_prompts( req.columns.get(col, []), col, - kwargs.get("append_payloads") if col == "text_column" else None, + content_extras, ) for col in ( "text_column", @@ -1061,11 +1044,12 @@ def format( # noqa: C901, PLR0912, PLR0915 if prefix: arguments.body["messages"].append({"role": "system", "content": prefix}) + content_extras = _get_content_extras(kwargs.get("extras")) prompts = [ self._format_prompts( data.columns.get(col, []), col, - kwargs.get("append_payloads") if col == "text_column" else None, + content_extras, ) for col in ( "text_column", @@ -1544,12 +1528,18 @@ def _ensure_tool_format(tool: dict[str, Any]) -> dict[str, Any]: return tool def _format_prompts( - self, column_data: list, column_type: str + self, + column_data: list, + column_type: str, + content_extras: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: formatted_data: list[dict[str, Any]] = [] for item in column_data: if column_type == "text_column": - formatted_data.append({"type": "input_text", "text": item}) + content = {"type": "input_text", "text": item} + if content_extras: + content.update(content_extras) + formatted_data.append(content) elif column_type == "image_column": formatted_data.append( { @@ -1629,8 +1619,13 @@ def _build_turn_input_items( # noqa: C901 items.append({"role": "assistant", "content": content}) else: # Standard or tool_call turn: user content. + content_extras = _get_content_extras(kwargs.get("extras")) prompts = [ - self._format_prompts(req.columns.get(col, []), col) + self._format_prompts( + req.columns.get(col, []), + col, + content_extras, + ) for col in ( "text_column", "image_column", @@ -1798,8 +1793,13 @@ def format( # noqa: C901 ) elif data.turn_type != "tool_response_injection": # Standard or tool_call turn: user content. + content_extras = _get_content_extras(kwargs.get("extras")) prompts = [ - self._format_prompts(data.columns.get(col, []), col) + self._format_prompts( + data.columns.get(col, []), + col, + content_extras, + ) for col in ( "text_column", "image_column", diff --git a/src/guidellm/cli/run.py b/src/guidellm/cli/run.py index 8d8cd554e..a8eefcee7 100644 --- a/src/guidellm/cli/run.py +++ b/src/guidellm/cli/run.py @@ -2,7 +2,6 @@ import asyncio from pathlib import Path -from typing import Any import click from pydantic import ValidationError @@ -29,22 +28,6 @@ ] -def _parse_append_payloads( - ctx: click.Context, param: click.Parameter, value: str | None -) -> dict[str, Any] | None: - """Parse and validate structured content payload fields from the CLI.""" - if value is None: - return None - parsed = cli_tools.parse_arguments(ctx, param, value) - if not isinstance(parsed, dict): - raise click.BadParameter( - "must be a JSON, YAML, or key=value object", - ctx=ctx, - param=param, - ) - return parsed - - @click.command( "run", help=( @@ -85,15 +68,6 @@ def _parse_append_payloads( ), ) @registry_options_from_model(model=BenchmarkArgs, group_key="spec") -@click.option( - "--append-payloads", - callback=_parse_append_payloads, - help=( - "Append key-value fields to structured content objects in chat completion " - "requests. Values override matching metadata from custom datasets. " - 'Example: `--append-payloads \'{"key":"value"}\'`' - ), -) @click.option( "--override", "benchmarks", @@ -132,12 +106,6 @@ def run(**kwargs): # noqa: C901, PLR0915 disable_console_interactive = ( kwargs.pop("disable_console_interactive", False) or disable_console ) - append_payloads = kwargs.pop("append_payloads", None) - if append_payloads is not None: - spec = kwargs.setdefault("spec", {}) - backend = spec.setdefault("backend", {}) - backend.setdefault("kind", "openai_http") - backend["append_payloads"] = append_payloads console = Console() if not disable_console else None if console: diff --git a/src/guidellm/data/finalizers/generative.py b/src/guidellm/data/finalizers/generative.py index 53acebcc6..90e9d1905 100644 --- a/src/guidellm/data/finalizers/generative.py +++ b/src/guidellm/data/finalizers/generative.py @@ -49,24 +49,6 @@ class GenerativeRequestFinalizer( def __init__(self, config: GenerativeRequestFinalizerArgs) -> None: self.config = config - @staticmethod - def _extract_prompt_text(prompt: Any) -> str: - """Extract metric-bearing text from a plain or structured prompt. - - :param prompt: A plain string or structured text content dictionary. - :return: The prompt text used for character and word metrics. - :raises ValueError: If the prompt does not contain valid text. - """ - if isinstance(prompt, str): - return prompt - if isinstance(prompt, dict) and isinstance(prompt.get("text"), str): - return prompt["text"] - if isinstance(prompt, dict): - raise ValueError( - "Structured text prompts must contain a string 'text' field" - ) - raise ValueError("Text prompts must be strings or structured content objects") - def __call__( self, items: list[dict[str, Any]] ) -> list[tuple[GenerationRequest, RequestSettings]]: @@ -124,11 +106,11 @@ def finalize_turn( # noqa: C901 PLR0912 input_metrics.add_text_metrics(prefix) # Count words in text prompts - for prompt in columns.get("text_column", []): - if not prompt: + for text in columns.get("text_column", []): + if not text: continue - input_metrics.add_text_metrics(self._extract_prompt_text(prompt)) + input_metrics.add_text_metrics(text) # Count pixels and bytes in images for image in columns.get("image_column", []): diff --git a/src/guidellm/schemas/request.py b/src/guidellm/schemas/request.py index 1367395ce..b3849d651 100644 --- a/src/guidellm/schemas/request.py +++ b/src/guidellm/schemas/request.py @@ -72,6 +72,16 @@ class GenerationRequestArguments(StandardBaseDict): default=None, description="Files to include in the request, if applicable.", ) + # used this explicitly here so backend ``extras.content`` is typed, validated, + # and included in generated schemas instead of existing only as an allowed + # arbitrary field on StandardBaseDict. + content: dict[str, Any] | None = Field( + default=None, + description=( + "Additional fields to include in generated text content objects, " + "if applicable." + ), + ) def model_combine( self, additional: GenerationRequestArguments | dict[str, Any] diff --git a/tests/unit/backends/openai/test_http.py b/tests/unit/backends/openai/test_http.py index abbf1c423..fc8a0f057 100644 --- a/tests/unit/backends/openai/test_http.py +++ b/tests/unit/backends/openai/test_http.py @@ -179,75 +179,33 @@ def test_server_history_with_responses_api(self): ) assert backend._args.server_history is True - @pytest.mark.sanity - def test_append_payloads_are_accepted_for_chat_completions(self): - """Chat backends accept structured content payload defaults. - - ## WRITTEN BY AI ## - """ - backend = _make_backend( - target="http://localhost:8000", - append_payloads={ - "metadata": {"category": "support"}, - "priority": 1, - }, - ) - - assert backend._args.append_payloads == { - "metadata": {"category": "support"}, - "priority": 1, - } - - @pytest.mark.sanity - @pytest.mark.parametrize("reserved_field", ["type", "text"]) - def test_append_payloads_reject_reserved_fields(self, reserved_field): - """Append payloads cannot replace GuideLLM-owned content fields. - - ## WRITTEN BY AI ## - """ - with pytest.raises(ValidationError, match="reserved content fields"): - _make_backend( - target="http://localhost:8000", - append_payloads={reserved_field: "replacement"}, - ) - - @pytest.mark.sanity - def test_append_payloads_reject_non_chat_endpoint(self): - """Append payloads are isolated from non-chat request handlers. - - ## WRITTEN BY AI ## - """ - with pytest.raises(ValidationError, match="only supported"): - _make_backend( - target="http://localhost:8000", - request_format="/v1/completions", - append_payloads={"priority": 1}, - ) - @pytest.mark.asyncio @pytest.mark.regression - async def test_append_payloads_are_forwarded_to_request_handler( - self, mock_request_handler + async def test_content_extras_are_forwarded_to_request_handler( + self, + mock_request_handler, ): - """The HTTP backend forwards append payloads through its handler boundary. + """The HTTP backend forwards content extras through its handler boundary. ## WRITTEN BY AI ## """ - payloads = { - "metadata": {"category": "support"}, - "priority": 1, - } + extras = GenerationRequestArguments( + content={ + "metadata": {"category": "support"}, + "priority": 1, + } + ) backend = _make_backend( target="http://localhost:8000", model="test-model", - append_payloads=payloads, + extras=extras, ) mock_handler, handler_patch = mock_request_handler with handler_patch: await backend._prepare_resolve_request(GenerationRequest()) - assert mock_handler.format.call_args.kwargs["append_payloads"] == payloads + assert mock_handler.format.call_args.kwargs["extras"] == extras @pytest.mark.smoke def test_factory_registration(self): diff --git a/tests/unit/backends/openai/test_request_handlers.py b/tests/unit/backends/openai/test_request_handlers.py index 85924abaf..85684a65b 100644 --- a/tests/unit/backends/openai/test_request_handlers.py +++ b/tests/unit/backends/openai/test_request_handlers.py @@ -29,7 +29,12 @@ GenerativeRequestFinalizer, GenerativeRequestFinalizerArgs, ) -from guidellm.schemas import GenerationRequest, GenerationResponse, UsageMetrics +from guidellm.schemas import ( + GenerationRequest, + GenerationRequestArguments, + GenerationResponse, + UsageMetrics, +) from guidellm.schemas.tool_call import ToolCall, ToolCallFunction from guidellm.settings import settings from guidellm.utils.registry import RegistryMixin @@ -954,64 +959,8 @@ def test_format_messages_text(self, valid_instances): assert result.body["messages"][0]["content"][1]["text"] == "How are you?" @pytest.mark.regression - def test_format_preserves_structured_text_metadata(self, valid_instances): - """Structured dataset prompts retain model-specific content fields. - - ## WRITTEN BY AI ## - """ - prompt = { - "type": "text", - "text": "Handle this request", - "metadata": {"category": "support"}, - "priority": 1, - } - data = GenerationRequest(columns={"text_column": [prompt]}) - - result = valid_instances.format(data) - - content = result.body["messages"][0]["content"][0] - assert content == prompt - assert content is not prompt - - @pytest.mark.regression - def test_append_payloads_override_structured_text_metadata(self, valid_instances): - """CLI payload values take precedence over custom dataset metadata. - - ## WRITTEN BY AI ## - """ - prompt = { - "type": "text", - "text": "Handle this request", - "priority": 1, - "metadata": {"source": "dataset"}, - "dataset_only": True, - } - data = GenerationRequest(columns={"text_column": [prompt]}) - - result = valid_instances.format( - data, - append_payloads={ - "priority": 2, - "metadata": {"source": "cli"}, - "cli_only": True, - }, - ) - - content = result.body["messages"][0]["content"][0] - assert content == { - "type": "text", - "text": "Handle this request", - "priority": 2, - "metadata": {"source": "cli"}, - "dataset_only": True, - "cli_only": True, - } - assert prompt["priority"] == 1 - assert prompt["metadata"] == {"source": "dataset"} - - @pytest.mark.regression - def test_append_payloads_enrich_plain_text_only(self, valid_instances): - """Append payloads enrich text without changing multimodal parts. + def test_content_extras_enrich_plain_text_only(self, valid_instances): + """Content extras enrich text without changing multimodal parts. ## WRITTEN BY AI ## """ @@ -1024,10 +973,12 @@ def test_append_payloads_enrich_plain_text_only(self, valid_instances): result = valid_instances.format( data, - append_payloads={ - "metadata": {"category": "vision"}, - "priority": 1, - }, + extras=GenerationRequestArguments( + content={ + "metadata": {"category": "vision"}, + "priority": 1, + } + ), ) text_content, image_content = result.body["messages"][0]["content"] @@ -1036,19 +987,6 @@ def test_append_payloads_enrich_plain_text_only(self, valid_instances): assert "metadata" not in image_content assert "priority" not in image_content - @pytest.mark.regression - def test_format_rejects_invalid_structured_text(self, valid_instances): - """Invalid structured prompt objects are rejected explicitly. - - ## WRITTEN BY AI ## - """ - data = GenerationRequest( - columns={"text_column": [{"type": "text", "text": 123}]} - ) - - with pytest.raises(ValueError, match="must contain a string 'text' field"): - valid_instances.format(data) - @pytest.mark.sanity def test_format_messages_prefix(self, valid_instances): """Test format method with prefix as system message. @@ -2723,45 +2661,23 @@ def test_chat_format_with_single_turn_history(self, valid_instances): assert messages[2]["role"] == "user" @pytest.mark.regression - def test_append_payloads_override_history_and_current_metadata( - self, valid_instances - ): - """Append payload precedence is consistent across conversation turns. + def test_content_extras_apply_to_history_and_current_turn(self, valid_instances): + """Content extras are applied consistently across conversation turns. ## WRITTEN BY AI ## """ - prev_request = GenerationRequest( - columns={ - "text_column": [ - { - "type": "text", - "text": "Previous", - "priority": 1, - } - ] - } - ) + prev_request = GenerationRequest(columns={"text_column": ["Previous"]}) prev_response = GenerationResponse( request_id="prev", request_args=None, text="Previous response", ) - data = GenerationRequest( - columns={ - "text_column": [ - { - "type": "text", - "text": "Current", - "priority": 2, - } - ] - } - ) + data = GenerationRequest(columns={"text_column": ["Current"]}) result = valid_instances.format( data, history=[(prev_request, prev_response)], - append_payloads={"priority": 3}, + extras=GenerationRequestArguments(content={"priority": 3}), ) messages = result.body["messages"] @@ -2955,6 +2871,56 @@ def test_factory_registration(self): handler = OpenAIRequestHandlerFactory.create("/v1/responses") assert isinstance(handler, ResponsesRequestHandler) + @pytest.mark.regression + def test_content_extras_enrich_text(self, valid_instances): + """Content extras are added to Responses API text content. + + ## WRITTEN BY AI ## + """ + data = GenerationRequest(columns={"text_column": ["Handle this request"]}) + + result = valid_instances.format( + data, + extras=GenerationRequestArguments( + content={ + "priority": 2, + "metadata": {"category": "support"}, + } + ), + ) + + content = result.body["input"][0]["content"][0] + assert content == { + "type": "input_text", + "text": "Handle this request", + "priority": 2, + "metadata": {"category": "support"}, + } + + @pytest.mark.regression + def test_content_extras_apply_to_history_and_current_turn(self, valid_instances): + """Content extras apply to all Responses API conversation turns. + + ## WRITTEN BY AI ## + """ + prev_request = GenerationRequest(columns={"text_column": ["Previous"]}) + prev_response = GenerationResponse( + request_id="prev", + request_args=None, + text="Previous response", + ) + data = GenerationRequest(columns={"text_column": ["Current"]}) + + result = valid_instances.format( + data, + history=[(prev_request, prev_response)], + extras=GenerationRequestArguments(content={"priority": 3}), + ) + + input_items = result.body["input"] + assert input_items[0]["content"][0]["priority"] == 3 + assert input_items[2]["content"][0]["priority"] == 3 + @pytest.mark.smoke def test_format_minimal(self, valid_instances): """ diff --git a/tests/unit/cli/test_run.py b/tests/unit/cli/test_run.py index 36e82ad95..1842b9a02 100644 --- a/tests/unit/cli/test_run.py +++ b/tests/unit/cli/test_run.py @@ -1,12 +1,9 @@ """Tests for ``guidellm run`` CLI error translation.""" -from unittest.mock import AsyncMock, patch - import pytest from click.testing import CliRunner from guidellm.__main__ import cli -from guidellm.backends.openai.http import OpenAIHTTPBackendArgs @pytest.mark.regression @@ -60,58 +57,3 @@ def test_run_allows_synthetic_text_without_output_tokens(): assert "Invalid value for --data" not in result.output assert "output_tokens" not in result.output - - -@pytest.mark.regression -def test_run_parses_append_payloads_into_openai_backend(): - """The top-level option is stored in the registered backend configuration. - - ## WRITTEN BY AI ## - """ - runner = CliRunner() - benchmark = AsyncMock() - - with patch("guidellm.cli.run.benchmark_generative_text", benchmark): - result = runner.invoke( - cli, - [ - "run", - "--backend", - "kind=openai_http,target=http://localhost:8000", - "--append-payloads", - '{"metadata":{"category":"support"},"priority":1}', - "--data", - "kind=synthetic_text,prompt_tokens=128", - "--disable-console", - ], - ) - - assert result.exit_code == 0, result.output - args = benchmark.await_args.kwargs["args"] - assert isinstance(args.spec.backend, OpenAIHTTPBackendArgs) - assert args.spec.backend.append_payloads == { - "metadata": {"category": "support"}, - "priority": 1, - } - - -@pytest.mark.regression -def test_run_rejects_non_object_append_payloads(): - """The CLI reports a clear error when append payloads is not an object. - - ## WRITTEN BY AI ## - """ - result = CliRunner().invoke( - cli, - [ - "run", - "--append-payloads", - '["not", "an", "object"]', - "--data", - "kind=synthetic_text,prompt_tokens=128", - ], - ) - - assert result.exit_code != 0 - assert "Invalid value for '--append-payloads'" in result.output - assert "must be a JSON, YAML, or key=value object" in result.output diff --git a/tests/unit/data/preprocessors/test_mappers.py b/tests/unit/data/preprocessors/test_mappers.py deleted file mode 100644 index f75660420..000000000 --- a/tests/unit/data/preprocessors/test_mappers.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Tests for generative dataset column mapping.""" - -from typing import Any - -import pytest -from datasets import Dataset - -from guidellm.data.preprocessors.mappers import ( - GenerativeColumnMapper, - GenerativeColumnMapperArgs, -) - - -def _structured_prompt( - text: str = "A structured prompt", - payload: dict[str, Any] | None = None, -) -> dict[str, Any]: - """Build a structured text prompt for mapper tests.""" - return { - "type": "text", - "text": text, - **(payload or {}), - } - - -@pytest.mark.regression -def test_generative_mapper_preserves_structured_prompt_objects(): - """Custom dataset prompt dictionaries pass through the mapper unchanged. - - ## WRITTEN BY AI ## - """ - prompt = _structured_prompt() - dataset = Dataset.from_list([{"prompt": prompt}]) - mapper = GenerativeColumnMapper(GenerativeColumnMapperArgs()) - mapper.setup_data([dataset]) - - result = mapper([{"dataset": dataset[0]}]) - - assert result == [{"text_column": [prompt]}] - - -@pytest.mark.smoke -def test_generative_mapper_keeps_plain_text_behavior(): - """Plain prompts and token-count columns retain their existing mapping. - - ## WRITTEN BY AI ## - """ - dataset = Dataset.from_list([{"prompt": "A plain prompt", "input_tokens_count": 3}]) - mapper = GenerativeColumnMapper(GenerativeColumnMapperArgs()) - mapper.setup_data([dataset]) - - result = mapper([{"dataset": dataset[0]}]) - - assert result == [ - { - "prompt_tokens_count_column": [3], - "text_column": ["A plain prompt"], - } - ] - - -@pytest.mark.sanity -def test_generative_mapper_preserves_structured_prompt_with_explicit_mapping(): - """Structured prompts work when the source column has a custom name. - - ## WRITTEN BY AI ## - """ - prompt = _structured_prompt(payload={"request_context": "custom-column"}) - dataset = Dataset.from_list([{"content_payload": prompt}]) - mapper = GenerativeColumnMapper( - GenerativeColumnMapperArgs(column_mappings={"text_column": "content_payload"}) - ) - mapper.setup_data([dataset]) - - result = mapper([{"dataset": dataset[0]}]) - - assert result == [{"text_column": [prompt]}] - - -@pytest.mark.regression -def test_generative_mapper_preserves_nested_and_optional_metadata(): - """Nested, list, boolean, and null metadata survive column mapping. - - ## WRITTEN BY AI ## - """ - prompt = _structured_prompt( - payload={ - "metadata": { - "category": "support", - "labels": ["billing", "priority"], - }, - "enabled": True, - "optional_value": None, - } - ) - dataset = Dataset.from_list([{"prompt": prompt}]) - mapper = GenerativeColumnMapper(GenerativeColumnMapperArgs()) - mapper.setup_data([dataset]) - - result = mapper([{"dataset": dataset[0]}]) - - mapped_prompt = result[0]["text_column"][0] - assert mapped_prompt["metadata"] == { - "category": "support", - "labels": ["billing", "priority"], - } - assert mapped_prompt["enabled"] is True - assert mapped_prompt["optional_value"] is None - - -@pytest.mark.regression -def test_generative_mapper_keeps_per_row_payloads_independent(): - """Metadata from one dataset row does not leak into another. - - ## WRITTEN BY AI ## - """ - prompts = [ - _structured_prompt(text="First item", payload={"request_id": "one"}), - _structured_prompt(text="Second item", payload={"request_id": "two"}), - ] - dataset = Dataset.from_list([{"prompt": prompt} for prompt in prompts]) - mapper = GenerativeColumnMapper(GenerativeColumnMapperArgs()) - mapper.setup_data([dataset]) - - results = [mapper([{"dataset": dataset[index]}]) for index in range(2)] - - assert results[0][0]["text_column"][0] == prompts[0] - assert results[1][0]["text_column"][0] == prompts[1] - assert results[0][0]["text_column"][0]["request_id"] == "one" - assert results[1][0]["text_column"][0]["request_id"] == "two" - - -@pytest.mark.regression -def test_generative_mapper_preserves_structured_multiturn_prompts(): - """Turn-suffixed structured prompt columns map to separate turns. - - ## WRITTEN BY AI ## - """ - first_turn = _structured_prompt(text="First turn", payload={"turn_id": 0}) - second_turn = _structured_prompt(text="Second turn", payload={"turn_id": 1}) - dataset = Dataset.from_list([{"prompt_0": first_turn, "prompt_1": second_turn}]) - mapper = GenerativeColumnMapper(GenerativeColumnMapperArgs()) - mapper.setup_data([dataset]) - - result = mapper([{"dataset": dataset[0]}]) - - assert result == [ - {"text_column": [first_turn]}, - {"text_column": [second_turn]}, - ] diff --git a/tests/unit/data/test_finalizers.py b/tests/unit/data/test_finalizers.py index bb84fec23..9aa69ecfd 100644 --- a/tests/unit/data/test_finalizers.py +++ b/tests/unit/data/test_finalizers.py @@ -147,41 +147,6 @@ def test_finalize_multi_value_text_columns(self, valid_instances): assert gen_req.input_metrics.text_words > 0 assert gen_req.input_metrics.text_characters > 0 - @pytest.mark.regression - def test_finalize_structured_text_preserves_metadata(self, valid_instances): - """Structured prompts retain metadata while metrics use their text. - - ## WRITTEN BY AI ## - """ - prompt = { - "type": "text", - "text": "The quick brown fox", - "metadata": {"category": "example"}, - "priority": 1, - } - - gen_req, _ = valid_instances.finalize_turn({"text_column": [prompt]}) - - assert gen_req.columns["text_column"] == [prompt] - assert gen_req.input_metrics.text_words == 4 - assert gen_req.input_metrics.text_characters == len(prompt["text"]) - - @pytest.mark.regression - def test_finalize_rejects_structured_text_without_string_text( - self, valid_instances - ): - """Malformed structured prompts fail with a useful validation message. - - ## WRITTEN BY AI ## - """ - with pytest.raises( - ValueError, - match="must contain a string 'text' field", - ): - valid_instances.finalize_turn( - {"text_column": [{"type": "text", "text": 123}]} - ) - @pytest.mark.sanity def test_finalize_multi_value_image_columns(self, valid_instances): """Test finalize sums image pixels and bytes across multiple images. diff --git a/tests/unit/schemas/test_request.py b/tests/unit/schemas/test_request.py index 423dde139..2105bfbf6 100644 --- a/tests/unit/schemas/test_request.py +++ b/tests/unit/schemas/test_request.py @@ -37,6 +37,7 @@ class TestGenerationRequestArguments: "params": {"limit": 10}, "body": {"prompt": "hello"}, "files": {"file": "data.txt"}, + "content": {"metadata": {"category": "support"}}, }, ], ids=["empty", "method_body", "method_headers_params", "all_fields"], @@ -57,7 +58,15 @@ def test_class_signatures(self): # Check fields fields = GenerationRequestArguments.model_fields - expected_fields = ["method", "stream", "headers", "params", "body", "files"] + expected_fields = [ + "method", + "stream", + "headers", + "params", + "body", + "files", + "content", + ] for field in expected_fields: assert field in fields @@ -72,7 +81,15 @@ def test_initialization(self, valid_instances): assert getattr(instance, key) == expected_value # Check defaults for fields not provided - for field in ["method", "stream", "headers", "params", "body", "files"]: + for field in [ + "method", + "stream", + "headers", + "params", + "body", + "files", + "content", + ]: if field not in constructor_args: assert getattr(instance, field) is None @@ -99,6 +116,10 @@ def test_invalid_initialization_values(self): with pytest.raises(ValidationError): GenerationRequestArguments(body="not_dict") + # Invalid content type + with pytest.raises(ValidationError): + GenerationRequestArguments(content="not_dict") + @pytest.mark.sanity def test_invalid_initialization_missing(self): """Test GenerationRequestArguments initialization without any fields.""" @@ -111,6 +132,7 @@ def test_invalid_initialization_missing(self): assert instance.params is None assert instance.body is None assert instance.files is None + assert instance.content is None @pytest.mark.smoke @pytest.mark.parametrize( From 4fd5f21650c8d041b785e7d962a56f43ed32b086 Mon Sep 17 00:00:00 2001 From: prasanna Date: Tue, 28 Jul 2026 09:11:24 +0530 Subject: [PATCH 3/4] updated the reviewed changes Signed-off-by: prasanna --- .../backends/openai/request_handlers.py | 8 ++--- src/guidellm/schemas/request.py | 3 -- .../backends/openai/test_request_handlers.py | 31 ++++++++++++------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/guidellm/backends/openai/request_handlers.py b/src/guidellm/backends/openai/request_handlers.py index 28d22837a..37bfdd451 100644 --- a/src/guidellm/backends/openai/request_handlers.py +++ b/src/guidellm/backends/openai/request_handlers.py @@ -194,18 +194,14 @@ def _check_streaming_error(data: Any) -> None: def _get_content_extras( - extras: GenerationRequestArguments | dict[str, Any] | None, + extras: GenerationRequestArguments | None, ) -> dict[str, Any] | None: """Extract content-object fields from generation request extras. :param extras: Additional generation request arguments. :return: Fields to merge into generated text content objects. """ - if isinstance(extras, GenerationRequestArguments): - return extras.content - if isinstance(extras, dict): - return extras.get("content") - return None + return extras.content if extras is not None else None class WSEventResult(Enum): diff --git a/src/guidellm/schemas/request.py b/src/guidellm/schemas/request.py index b3849d651..370291b3f 100644 --- a/src/guidellm/schemas/request.py +++ b/src/guidellm/schemas/request.py @@ -72,9 +72,6 @@ class GenerationRequestArguments(StandardBaseDict): default=None, description="Files to include in the request, if applicable.", ) - # used this explicitly here so backend ``extras.content`` is typed, validated, - # and included in generated schemas instead of existing only as an allowed - # arbitrary field on StandardBaseDict. content: dict[str, Any] | None = Field( default=None, description=( diff --git a/tests/unit/backends/openai/test_request_handlers.py b/tests/unit/backends/openai/test_request_handlers.py index 85684a65b..863efb5e8 100644 --- a/tests/unit/backends/openai/test_request_handlers.py +++ b/tests/unit/backends/openai/test_request_handlers.py @@ -507,7 +507,7 @@ def test_format_extras(self, valid_instances): """ instance = valid_instances data = GenerationRequest() - extras = {"body": {"temperature": 0.7, "top_p": 0.9}} + extras = GenerationRequestArguments(body={"temperature": 0.7, "top_p": 0.9}) result = instance.format(data, extras=extras) @@ -930,7 +930,7 @@ def test_format_extras(self, valid_instances): """ instance = valid_instances data = GenerationRequest() - extras = {"body": {"temperature": 0.5, "top_k": 40}} + extras = GenerationRequestArguments(body={"temperature": 0.5, "top_k": 40}) result = instance.format(data, extras=extras) @@ -2095,7 +2095,10 @@ def test_format_strips_tool_choice_without_tools(self, valid_instances): turn_type="standard", ) - result = instance.format(data, extras={"body": {"tool_choice": "required"}}) + result = instance.format( + data, + extras=GenerationRequestArguments(body={"tool_choice": "required"}), + ) assert "tool_choice" not in result.body assert "tools" not in result.body @@ -2307,7 +2310,7 @@ def test_format_extras(self, valid_instances): ] }, ) - extras = {"body": {"language": "en", "temperature": 0.0}} + extras = GenerationRequestArguments(body={"language": "en", "temperature": 0.0}) result = instance.format(data, extras=extras) @@ -4455,7 +4458,10 @@ def test_format_tool_choice_none_on_non_tool_turn(self, valid_instances): tools = [{"type": "function", "function": {"name": "fn", "parameters": {}}}] data = GenerationRequest(turn_type="standard") - result = instance.format(data, extras={"body": {"tools": tools}}) + result = instance.format( + data, + extras=GenerationRequestArguments(body={"tools": tools}), + ) assert result.body["tools"] == tools assert result.body["tool_choice"] == "none" @@ -4474,7 +4480,10 @@ def test_format_strips_tool_choice_without_tools(self, valid_instances): turn_type="standard", ) - result = instance.format(data, extras={"body": {"tool_choice": "required"}}) + result = instance.format( + data, + extras=GenerationRequestArguments(body={"tool_choice": "required"}), + ) assert "tool_choice" not in result.body assert "tools" not in result.body @@ -4752,7 +4761,7 @@ def test_format_extras(self, valid_instances): """ instance = valid_instances data = GenerationRequest() - extras = {"body": {"temperature": 0.5, "top_k": 40}} + extras = GenerationRequestArguments(body={"temperature": 0.5, "top_k": 40}) result = instance.format(data, extras=extras) @@ -4976,7 +4985,7 @@ def test_format_with_extras(self, valid_instances): """ instance = valid_instances data = GenerationRequest() - extras = {"body": {"user": "test-user"}} + extras = GenerationRequestArguments(body={"user": "test-user"}) result = instance.format(data, extras=extras) @@ -5081,7 +5090,7 @@ def test_tool_choice_none_when_expects_false(self, handler): }, turn_type="standard", ) - extras = {"body": {"tool_choice": "required"}} + extras = GenerationRequestArguments(body={"tool_choice": "required"}) result = handler.format(data, extras=extras) assert result.body["tool_choice"] == "none" @@ -5100,7 +5109,7 @@ def test_tool_choice_preserved_when_expects_true(self, handler): }, turn_type="client_tool_call", ) - extras = {"body": {"tool_choice": "required"}} + extras = GenerationRequestArguments(body={"tool_choice": "required"}) result = handler.format(data, extras=extras) assert result.body["tool_choice"] == "required" @@ -5119,7 +5128,7 @@ def test_auto_tool_choice_preserved_when_expects_true(self, handler): }, turn_type="client_tool_call", ) - extras = {"body": {"tool_choice": "auto"}} + extras = GenerationRequestArguments(body={"tool_choice": "auto"}) result = handler.format(data, extras=extras) assert result.body["tool_choice"] == "auto" From cec92212fefd219e2d17cac5dd7799fb38e02a4a Mon Sep 17 00:00:00 2001 From: prasanna Date: Wed, 29 Jul 2026 12:24:05 +0530 Subject: [PATCH 4/4] reviewed changes removing the helper function Signed-off-by: prasanna --- .../backends/openai/request_handlers.py | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/src/guidellm/backends/openai/request_handlers.py b/src/guidellm/backends/openai/request_handlers.py index 37bfdd451..ed9ba11bf 100644 --- a/src/guidellm/backends/openai/request_handlers.py +++ b/src/guidellm/backends/openai/request_handlers.py @@ -193,17 +193,6 @@ def _check_streaming_error(data: Any) -> None: raise ValueError(f"Streaming response returned an error: {message}") -def _get_content_extras( - extras: GenerationRequestArguments | None, -) -> dict[str, Any] | None: - """Extract content-object fields from generation request extras. - - :param extras: Additional generation request arguments. - :return: Fields to merge into generated text content objects. - """ - return extras.content if extras is not None else None - - class WSEventResult(Enum): """Classification of a processed WebSocket streaming event.""" @@ -927,7 +916,8 @@ def _build_turn_messages( # noqa: C901 if prefix: messages.append({"role": "system", "content": prefix}) - content_extras = _get_content_extras(kwargs.get("extras")) + extras = kwargs.get("extras") + content_extras = extras.content if extras is not None else None prompts = [ self._format_prompts( req.columns.get(col, []), @@ -1040,7 +1030,8 @@ def format( # noqa: C901, PLR0912, PLR0915 if prefix: arguments.body["messages"].append({"role": "system", "content": prefix}) - content_extras = _get_content_extras(kwargs.get("extras")) + extras = kwargs.get("extras") + content_extras = extras.content if extras is not None else None prompts = [ self._format_prompts( data.columns.get(col, []), @@ -1615,7 +1606,8 @@ def _build_turn_input_items( # noqa: C901 items.append({"role": "assistant", "content": content}) else: # Standard or tool_call turn: user content. - content_extras = _get_content_extras(kwargs.get("extras")) + extras = kwargs.get("extras") + content_extras = extras.content if extras is not None else None prompts = [ self._format_prompts( req.columns.get(col, []), @@ -1789,7 +1781,8 @@ def format( # noqa: C901 ) elif data.turn_type != "tool_response_injection": # Standard or tool_call turn: user content. - content_extras = _get_content_extras(kwargs.get("extras")) + extras = kwargs.get("extras") + content_extras = extras.content if extras is not None else None prompts = [ self._format_prompts( data.columns.get(col, []),