From f51c28fc3cd640cb3e71e247af4ba15faf77982d Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 12 Mar 2026 11:35:27 -0700 Subject: [PATCH 1/4] Add support to new openai text to image model --- .../services/open_ai_text_to_image_base.py | 4 +- .../services/test_openai_text_to_image.py | 106 ++++++++++++++++++ 2 files changed, 108 insertions(+), 2 deletions(-) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_to_image_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_to_image_base.py index c3161043384e..011a09ec10fa 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_to_image_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_to_image_base.py @@ -68,10 +68,10 @@ async def generate_image( response = await self._send_request(settings) assert isinstance(response, ImagesResponse) # nosec - if not response.data or not response.data[0].url: + if not response.data or not (response.data[0].url or response.data[0].b64_json): raise ServiceResponseException("Failed to generate image.") - return response.data[0].url + return response.data[0].url or response.data[0].b64_json # type: ignore[return-value] async def generate_images( self, diff --git a/python/tests/unit/connectors/ai/open_ai/services/test_openai_text_to_image.py b/python/tests/unit/connectors/ai/open_ai/services/test_openai_text_to_image.py index 6d5845994fda..30df91eef3fe 100644 --- a/python/tests/unit/connectors/ai/open_ai/services/test_openai_text_to_image.py +++ b/python/tests/unit/connectors/ai/open_ai/services/test_openai_text_to_image.py @@ -267,3 +267,109 @@ async def test_edit_image_invalid_n_parameter(): OpenAITextToImageExecutionSettings(n=0) with pytest.raises(pydantic.ValidationError): OpenAITextToImageExecutionSettings(n=11) + + +@pytest.mark.asyncio +async def test_generate_images_empty_prompt(openai_unit_test_env): + """Test that empty prompt raises ServiceInvalidRequestError.""" + service = OpenAITextToImage(ai_model_id=openai_unit_test_env["OPENAI_TEXT_TO_IMAGE_MODEL_ID"]) + with pytest.raises(ServiceInvalidRequestError): + await service.generate_images("") + + +@patch.object(OpenAITextToImageBase, "_send_request", new_callable=AsyncMock) +async def test_generate_images_no_result(mock_generate, openai_unit_test_env): + """Test that empty response data raises ServiceResponseException.""" + mock_generate.return_value = ImagesResponse(created=0, data=[], usage=None) + service = OpenAITextToImage(ai_model_id=openai_unit_test_env["OPENAI_TEXT_TO_IMAGE_MODEL_ID"]) + with pytest.raises(ServiceResponseException): + await service.generate_images("prompt") + + +@patch.object(OpenAITextToImageBase, "_send_request", new_callable=AsyncMock) +async def test_generate_images_b64_json_response(mock_generate, openai_unit_test_env): + """Test that generate_images returns b64_json when url is not present.""" + mock_generate.return_value = ImagesResponse(created=1, data=[Image(b64_json="base64encodeddata")], usage=None) + service = OpenAITextToImage(ai_model_id=openai_unit_test_env["OPENAI_TEXT_TO_IMAGE_MODEL_ID"]) + result = await service.generate_images("prompt") + assert result == ["base64encodeddata"] + + +@patch.object(OpenAITextToImageBase, "_send_request", new_callable=AsyncMock) +async def test_generate_images_mixed_url_and_b64_response(mock_generate, openai_unit_test_env): + """Test that generate_images handles mixed url and b64_json responses.""" + mock_generate.return_value = ImagesResponse( + created=2, + data=[Image(url="http://example.com/img1.png"), Image(b64_json="base64data")], + usage=None, + ) + service = OpenAITextToImage(ai_model_id=openai_unit_test_env["OPENAI_TEXT_TO_IMAGE_MODEL_ID"]) + result = await service.generate_images("prompt") + assert result == ["http://example.com/img1.png", "base64data"] + + +@patch.object(OpenAITextToImageBase, "_send_request", new_callable=AsyncMock) +async def test_generate_images_with_default_settings(mock_generate, openai_unit_test_env): + """Test that generate_images works when no settings are provided.""" + mock_generate.return_value = ImagesResponse(created=1, data=[Image(url="url")], usage=None) + service = OpenAITextToImage(ai_model_id=openai_unit_test_env["OPENAI_TEXT_TO_IMAGE_MODEL_ID"]) + result = await service.generate_images("a beautiful sunset") + assert result == ["url"] + mock_generate.assert_awaited_once() + + +@patch.object(OpenAITextToImageBase, "_send_request", new_callable=AsyncMock) +async def test_generate_images_no_valid_image_data(mock_generate, openai_unit_test_env): + """Test that generate_images raises error when images have neither url nor b64_json.""" + mock_generate.return_value = ImagesResponse(created=1, data=[Image()], usage=None) + service = OpenAITextToImage(ai_model_id=openai_unit_test_env["OPENAI_TEXT_TO_IMAGE_MODEL_ID"]) + with pytest.raises(ServiceResponseException, match="No valid image data found"): + await service.generate_images("prompt") + + +@pytest.mark.asyncio +async def test_edit_image_neither_path_nor_file(openai_unit_test_env): + """Test that providing neither image_paths nor image_files raises ServiceInvalidRequestError.""" + service = OpenAITextToImage(ai_model_id=openai_unit_test_env["OPENAI_TEXT_TO_IMAGE_MODEL_ID"]) + with pytest.raises(ServiceInvalidRequestError): + await service.edit_image(prompt="edit this") + + +@patch.object(OpenAITextToImageBase, "_send_image_edit_request", new_callable=AsyncMock) +async def test_edit_image_b64_json_response(mock_edit, openai_unit_test_env): + """Test editing an image returns b64_json when url is not present.""" + mock_edit.return_value = ImagesResponse(created=1, data=[Image(b64_json="edited_b64")], usage=None) + service = OpenAITextToImage(ai_model_id=openai_unit_test_env["OPENAI_TEXT_TO_IMAGE_MODEL_ID"]) + result = await service.edit_image( + prompt="edit this image", + image_paths=[sample_img], + ) + assert result == ["edited_b64"] + + +@patch.object(OpenAITextToImageBase, "_send_image_edit_request", new_callable=AsyncMock) +async def test_edit_image_mixed_response(mock_edit, openai_unit_test_env): + """Test editing images handles mixed b64_json and url responses.""" + mock_edit.return_value = ImagesResponse( + created=2, + data=[Image(b64_json="b64data"), Image(url="http://example.com/edited.png")], + usage=None, + ) + service = OpenAITextToImage(ai_model_id=openai_unit_test_env["OPENAI_TEXT_TO_IMAGE_MODEL_ID"]) + result = await service.edit_image( + prompt="edit these images", + image_paths=[sample_img], + ) + assert result == ["b64data", "http://example.com/edited.png"] + + +@patch.object(OpenAITextToImageBase, "_send_image_edit_request", new_callable=AsyncMock) +async def test_edit_image_response_no_data_attribute(mock_edit, openai_unit_test_env): + """Test that edit_image raises error when response has no valid data.""" + mock_edit.return_value = ImagesResponse(created=1, data=None, usage=None) + service = OpenAITextToImage(ai_model_id=openai_unit_test_env["OPENAI_TEXT_TO_IMAGE_MODEL_ID"]) + with pytest.raises(ServiceResponseException): + await service.edit_image( + prompt="edit", + image_paths=[sample_img], + ) From 5fc0c690b02119984ac882423b904a71da16c5d9 Mon Sep 17 00:00:00 2001 From: Evan Mattson Date: Fri, 13 Mar 2026 16:20:30 +0900 Subject: [PATCH 2/4] Python: Remove unsupported style and quality params from image_generation sample The gpt-image-1 model does not support the 'style' or 'quality' (with value 'hd') parameters that were specific to DALL-E 3. --- python/samples/concepts/images/image_generation.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/samples/concepts/images/image_generation.py b/python/samples/concepts/images/image_generation.py index df6e71a9b477..a97f141d9dc8 100644 --- a/python/samples/concepts/images/image_generation.py +++ b/python/samples/concepts/images/image_generation.py @@ -24,9 +24,7 @@ async def main(): kernel.add_service(dalle3) kernel.add_service(OpenAIChatCompletion(service_id="default")) - image = await dalle3.generate_image( - description="a painting of a flower vase", width=1024, height=1024, quality="hd", style="vivid" - ) + image = await dalle3.generate_image(description="a painting of a flower vase", width=1024, height=1024) print(image) if pil_available: img = Image.open(urlopen(image)) # nosec From d4b15c1b936f3690167e7cba61452d8746130d9a Mon Sep 17 00:00:00 2001 From: Evan Mattson Date: Fri, 13 Mar 2026 17:19:37 +0900 Subject: [PATCH 3/4] Python: Update image_generation sample to handle base64 response from gpt-image-1 DALL-E 3 is being retired in favor of gpt-image-1 which returns base64 data instead of URLs. Update the sample to decode base64 and use ImageContent with data instead of uri. --- .../samples/concepts/images/image_generation.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/python/samples/concepts/images/image_generation.py b/python/samples/concepts/images/image_generation.py index a97f141d9dc8..74f65b9f9db8 100644 --- a/python/samples/concepts/images/image_generation.py +++ b/python/samples/concepts/images/image_generation.py @@ -1,7 +1,8 @@ # Copyright (c) Microsoft. All rights reserved. import asyncio -from urllib.request import urlopen +import base64 +from io import BytesIO from semantic_kernel.prompt_template import PromptTemplateConfig @@ -20,14 +21,14 @@ async def main(): kernel = Kernel() - dalle3 = OpenAITextToImage() - kernel.add_service(dalle3) + service = OpenAITextToImage() + kernel.add_service(service) kernel.add_service(OpenAIChatCompletion(service_id="default")) - image = await dalle3.generate_image(description="a painting of a flower vase", width=1024, height=1024) - print(image) + image_b64 = await service.generate_image(description="a painting of a flower vase", width=1024, height=1024) + if pil_available: - img = Image.open(urlopen(image)) # nosec + img = Image.open(BytesIO(base64.b64decode(image_b64))) img.show() result = await kernel.invoke_prompt( @@ -40,7 +41,7 @@ async def main(): role="user", items=[ TextContent(text="What is in this image?"), - ImageContent(uri=image), + ImageContent(data=image_b64, data_format="base64", mime_type="image/png"), ], ) ] From 63ade4363209bf22e629311953dff2ec60e07855 Mon Sep 17 00:00:00 2001 From: Evan Mattson Date: Fri, 13 Mar 2026 17:26:35 +0900 Subject: [PATCH 4/4] Python: Remove deprecated style param and constrain quality values for gpt-image-1 DALL-E 3 is being retired. The 'style' parameter is not supported by gpt-image-1 and 'quality' only accepts 'high', 'medium', or 'low'. --- .../open_ai_text_to_image_execution_settings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_text_to_image_execution_settings.py b/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_text_to_image_execution_settings.py index 3c4d45122940..e3a11cf1fbe1 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_text_to_image_execution_settings.py +++ b/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_text_to_image_execution_settings.py @@ -38,8 +38,7 @@ class OpenAITextToImageExecutionSettings(PromptExecutionSettings): prompt: str | None = None ai_model_id: str | None = Field(default=None, serialization_alias="model") size: ImageSize | None = None - quality: str | None = None - style: str | None = None + quality: Literal["high", "medium", "low"] | None = None output_compression: int | None = None background: Literal["transparent", "opaque", "auto"] | None = None n: int | None = Field(default=1, ge=1, le=10)