.Net: feat(connectors): Support ImageContent in tool/function results

dotnet/src/Agents/Bedrock/Extensions/BedrockAgentInvokeExtensions.cs

@@ -214,12 +214,28 @@ private static SessionState CreateSessionStateWithFunctionResults(List<FunctionR
                                 Function = functionResult.FunctionName,
                                 ResponseBody = new Dictionary<string, ContentBody>
                                 {
-                                    { "TEXT", new ContentBody() { Body = FunctionCallsProcessor.ProcessFunctionResult(functionResult.Result ?? string.Empty) } }
+                                    { "TEXT", new ContentBody() { Body = GetFunctionResultAsString(functionResult.Result) } }
                                 }
                             }
                         };
                     }
                 )],
             };
     }
+
+    /// <summary>
+    /// Processes a function result and returns a string representation.
+    /// Bedrock does not support multimodal tool results, so ImageContent returns an error message.
+    /// </summary>
+    private static string GetFunctionResultAsString(object? result)
+    {
+        var processed = FunctionCallsProcessor.ProcessFunctionResult(result ?? string.Empty);
+
+        if (processed is ImageContent)
+        {
+            return FunctionCallsProcessor.ImageContentNotSupportedErrorMessage;
+        }
+
+        return (string?)processed ?? string.Empty;
+    }
 }

dotnet/src/Agents/OpenAI/Internal/AssistantMessageFactory.cs

@@ -71,8 +71,24 @@ public static IEnumerable<MessageContent> GetMessageContents(ChatMessageContent
             else if (content is FunctionResultContent resultContent && resultContent.Result != null && !hasTextContent)
             {
                 // Only convert a function result when text-content is not already present
-                yield return MessageContent.FromText(FunctionCallsProcessor.ProcessFunctionResult(resultContent.Result));
+                yield return MessageContent.FromText(GetFunctionResultAsString(resultContent.Result));
             }
         }
     }
+
+    /// <summary>
+    /// Processes a function result and returns a string representation.
+    /// OpenAI Assistants do not support multimodal tool results, so ImageContent returns an error message.
+    /// </summary>
+    private static string GetFunctionResultAsString(object result)
+    {
+        var processed = FunctionCallsProcessor.ProcessFunctionResult(result);
+
+        if (processed is ImageContent)
+        {
+            return FunctionCallsProcessor.ImageContentNotSupportedErrorMessage;
+        }
+
+        return (string?)processed ?? string.Empty;
+    }
 }

dotnet/src/Agents/OpenAI/Internal/ResponseThreadActions.cs

@@ -93,7 +93,7 @@ await functionProcessor.InvokeFunctionCallsAsync(
                     agent.GetKernel(options),
                     isStreaming: false,
                     cancellationToken).ConfigureAwait(false);
-            var functionOutputItems = functionResults.Select(fr => ResponseItem.CreateFunctionCallOutputItem(fr.CallId, fr.Result?.ToString() ?? string.Empty)).ToList();
+            var functionOutputItems = functionResults.Select(fr => ResponseItem.CreateFunctionCallOutputItem(fr.CallId, GetFunctionResultAsString(fr.Result))).ToList();
 
             // If store is enabled we only need to send the function output items
             if (agent.StoreEnabled)
@@ -267,7 +267,7 @@ await functionProcessor.InvokeFunctionCallsAsync(
                     agent.GetKernel(options),
                     isStreaming: true,
                     cancellationToken).ConfigureAwait(false);
-            var functionOutputItems = functionResults.Select(fr => ResponseItem.CreateFunctionCallOutputItem(fr.CallId, fr.Result?.ToString() ?? string.Empty)).ToList();
+            var functionOutputItems = functionResults.Select(fr => ResponseItem.CreateFunctionCallOutputItem(fr.CallId, GetFunctionResultAsString(fr.Result))).ToList();
 
             // If store is enabled we only need to send the function output items
             if (agent.StoreEnabled)
@@ -318,6 +318,22 @@ private static void ThrowIfIncompleteOrFailed(OpenAIResponseAgent agent, Respons
         }
     }
 
+    /// <summary>
+    /// Processes a function result and returns a string representation.
+    /// The OpenAI Responses API does not support multimodal tool results, so ImageContent returns an error message.
+    /// </summary>
+    internal static string GetFunctionResultAsString(object? result)
+    {
+        var processed = FunctionCallsProcessor.ProcessFunctionResult(result ?? string.Empty);
+
+        if (processed is ImageContent)
+        {
+            return FunctionCallsProcessor.ImageContentNotSupportedErrorMessage;
+        }
+
+        return (string?)processed ?? string.Empty;
+    }
+
     /// <summary>POCO representing function calling info.</summary>
     /// <remarks>Used to concatenation information for a single function call from across multiple streaming updates.</remarks>
     private sealed class FunctionCallInfo(FunctionCallResponseItem item)

dotnet/src/Agents/UnitTests/OpenAI/Internal/AssistantMessageFactoryTests.cs

@@ -5,6 +5,7 @@
 using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.Agents.OpenAI.Internal;
 using Microsoft.SemanticKernel.ChatCompletion;
+
 using OpenAI.Assistants;
 using Xunit;
 
@@ -207,4 +208,28 @@ public void VerifyAssistantMessageAdapterGetMessageWithAll()
         Assert.NotNull(contents);
         Assert.Equal(3, contents.Length);
     }
+
+    /// <summary>
+    /// Verify that ImageContent in FunctionResultContent returns error message
+    /// since OpenAI Assistants do not support multimodal tool results.
+    /// </summary>
+    [Fact]
+    public void VerifyAssistantMessageAdapterGetMessageWithImageContentInFunctionResult()
+    {
+        // Arrange: Create a FunctionResultContent containing ImageContent
+        var imageData = new ReadOnlyMemory<byte>([0x89, 0x50, 0x4E, 0x47]); // PNG magic bytes
+        var imageContent = new ImageContent(imageData, "image/png");
+        var functionResultContent = new FunctionResultContent("TestFunction", "TestPlugin", "call-id", imageContent);
+        ChatMessageContent message = new(AuthorRole.Tool, items: [functionResultContent]);
+
+        // Act
+        MessageContent[] contents = AssistantMessageFactory.GetMessageContents(message).ToArray();
+
+        // Assert: Should return error message since OpenAI Assistants don't support multimodal tool results
+        Assert.NotNull(contents);
+        Assert.Single(contents);
+        Assert.NotNull(contents.Single().Text);
+        // Expected error message from FunctionCallsProcessor.ImageContentNotSupportedErrorMessage
+        Assert.Equal("Error: This model does not support image content in tool results.", contents.Single().Text);
+    }
 }

dotnet/src/Agents/UnitTests/OpenAI/Internal/ResponseThreadActionsTests.cs

@@ -0,0 +1,64 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Agents.OpenAI.Internal;
+using Xunit;
+
+namespace SemanticKernel.Agents.UnitTests.OpenAI.Internal;
+
+/// <summary>
+/// Unit tests for <see cref="ResponseThreadActions"/>.
+/// </summary>
+public class ResponseThreadActionsTests
+{
+    /// <summary>
+    /// Verify that <see cref="ResponseThreadActions.GetFunctionResultAsString(object?)"/> returns the
+    /// shared <c>ImageContentNotSupportedErrorMessage</c> when the function result is an
+    /// <see cref="ImageContent"/>, since the OpenAI Responses API does not support multimodal tool results.
+    /// </summary>
+    [Fact]
+    public void VerifyResponseThreadActionsGetFunctionResultAsStringReturnsErrorMessageForImageContent()
+    {
+        // Arrange: Create an ImageContent with binary data
+        var imageData = new ReadOnlyMemory<byte>([0x89, 0x50, 0x4E, 0x47]); // PNG magic bytes
+        var imageContent = new ImageContent(imageData, "image/png");
+
+        // Act
+        string result = ResponseThreadActions.GetFunctionResultAsString(imageContent);
+
+        // Assert
+        Assert.Equal("Error: This model does not support image content in tool results.", result);
+    }
+
+    /// <summary>
+    /// Verify that <see cref="ResponseThreadActions.GetFunctionResultAsString(object?)"/> returns the
+    /// original string verbatim when the function result is a string.
+    /// </summary>
+    [Fact]
+    public void VerifyResponseThreadActionsGetFunctionResultAsStringReturnsStringVerbatim()
+    {
+        // Arrange
+        const string Expected = "tool result text";
+
+        // Act
+        string result = ResponseThreadActions.GetFunctionResultAsString(Expected);
+
+        // Assert
+        Assert.Equal(Expected, result);
+    }
+
+    /// <summary>
+    /// Verify that <see cref="ResponseThreadActions.GetFunctionResultAsString(object?)"/> returns
+    /// <see cref="string.Empty"/> when the function result is <see langword="null"/>.
+    /// </summary>
+    [Fact]
+    public void VerifyResponseThreadActionsGetFunctionResultAsStringReturnsEmptyForNull()
+    {
+        // Act
+        string result = ResponseThreadActions.GetFunctionResultAsString(null);
+
+        // Assert
+        Assert.Equal(string.Empty, result);
+    }
+}

dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs

@@ -781,6 +781,71 @@ public void FromChatHistoryMultiTurnConversationPreservesAllRoles()
         Assert.Equal("assistant-message-2", request.Contents[3].Parts![0].Text);
     }
 
+    [Fact]
+    public void FromChatHistoryImageContentInToolResultCreatesInlineDataPart()
+    {
+        // Arrange
+        ChatHistory chatHistory = [];
+        var imageBytes = new byte[] { 0x89, 0x50, 0x4E, 0x47 }; // PNG magic bytes
+        var imageContent = new ImageContent(imageBytes, "image/png");
+        var kernelFunction = KernelFunctionFactory.CreateFromMethod(() => imageContent);
+        var toolCall = new GeminiFunctionToolCall(new GeminiPart.FunctionCallPart { FunctionName = "capture-screenshot" });
+        GeminiFunctionToolResult toolCallResult = new(toolCall, new FunctionResult(kernelFunction, imageContent));
+        chatHistory.Add(new GeminiChatMessageContent(AuthorRole.Tool, string.Empty, "modelId", toolCallResult));
+        var executionSettings = new GeminiPromptExecutionSettings();
+
+        // Act
+        var request = GeminiRequest.FromChatHistoryAndExecutionSettings(chatHistory, executionSettings);
+
+        // Assert
+        Assert.Single(request.Contents);
+        var part = request.Contents[0].Parts![0];
+        Assert.NotNull(part.FunctionResponse);
+        Assert.Equal("capture-screenshot", part.FunctionResponse.FunctionName);
+        Assert.NotNull(part.FunctionResponse.Parts);
+        Assert.Single(part.FunctionResponse.Parts);
+        Assert.NotNull(part.FunctionResponse.Parts[0].InlineData);
+        Assert.Equal("image/png", part.FunctionResponse.Parts[0].InlineData!.MimeType);
+        Assert.Equal(Convert.ToBase64String(imageBytes), part.FunctionResponse.Parts[0].InlineData!.InlineData);
+    }
+
+    [Fact]
+    public void FromChatHistoryImageContentWithoutDataThrowsInvalidOperationException()
+    {
+        // Arrange
+        ChatHistory chatHistory = [];
+        var imageContent = new ImageContent(new Uri("https://example.com/image.png")) { MimeType = "image/png" };
+        var kernelFunction = KernelFunctionFactory.CreateFromMethod(() => imageContent);
+        var toolCall = new GeminiFunctionToolCall(new GeminiPart.FunctionCallPart { FunctionName = "capture-screenshot" });
+        GeminiFunctionToolResult toolCallResult = new(toolCall, new FunctionResult(kernelFunction, imageContent));
+        chatHistory.Add(new GeminiChatMessageContent(AuthorRole.Tool, string.Empty, "modelId", toolCallResult));
+        var executionSettings = new GeminiPromptExecutionSettings();
+
+        // Act & Assert
+        var exception = Assert.Throws<InvalidOperationException>(
+            () => GeminiRequest.FromChatHistoryAndExecutionSettings(chatHistory, executionSettings));
+        Assert.Equal("ImageContent in function result must contain binary data.", exception.Message);
+    }
+
+    [Fact]
+    public void FromChatHistoryImageContentWithoutMimeTypeThrowsInvalidOperationException()
+    {
+        // Arrange
+        ChatHistory chatHistory = [];
+        ReadOnlyMemory<byte> imageBytes = new byte[] { 0x89, 0x50, 0x4E, 0x47 };
+        var imageContent = new ImageContent(imageBytes, mimeType: null); // No MimeType
+        var kernelFunction = KernelFunctionFactory.CreateFromMethod(() => imageContent);
+        var toolCall = new GeminiFunctionToolCall(new GeminiPart.FunctionCallPart { FunctionName = "capture-screenshot" });
+        GeminiFunctionToolResult toolCallResult = new(toolCall, new FunctionResult(kernelFunction, imageContent));
+        chatHistory.Add(new GeminiChatMessageContent(AuthorRole.Tool, string.Empty, "modelId", toolCallResult));
+        var executionSettings = new GeminiPromptExecutionSettings();
+
+        // Act & Assert
+        var exception = Assert.Throws<InvalidOperationException>(
+            () => GeminiRequest.FromChatHistoryAndExecutionSettings(chatHistory, executionSettings));
+        Assert.Equal("Image content MimeType is empty.", exception.Message);
+    }
+
     [Fact]
     public void FromChatHistoryToolCallsWithThoughtSignatureIncludesSignatureInRequest()
     {

dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiPart.cs

@@ -185,6 +185,14 @@ internal sealed class FunctionResponsePart
         [JsonRequired]
         public FunctionResponseEntity Response { get; set; } = null!;
 
+        /// <summary>
+        /// Optional. Nested parts for multimodal function responses (Gemini 3+ only).
+        /// Contains inlineData with image/binary data as part of tool results.
+        /// </summary>
+        [JsonPropertyName("parts")]
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+        public FunctionResponsePartContent[]? Parts { get; set; }
+
         internal sealed class FunctionResponseEntity
         {
             [JsonConstructor]
@@ -202,5 +210,16 @@ public FunctionResponseEntity(object? response)
             [JsonRequired]
             public JsonNode Arguments { get; set; } = null!;
         }
+
+        /// <summary>
+        /// Represents a part within a Gemini function response (for multimodal content).
+        /// Used in Gemini 3+ to include images/binary data as part of tool results.
+        /// </summary>
+        internal sealed class FunctionResponsePartContent
+        {
+            [JsonPropertyName("inlineData")]
+            [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+            public InlineDataPart? InlineData { get; set; }
+        }
     }
 }

dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs

@@ -23,6 +23,13 @@ internal sealed class GeminiRequest
         }
     };
 
+    /// <summary>
+    /// Synthetic envelope used as the <c>functionResponse.response</c> body when emitting a multimodal
+    /// (image) tool result. The actual image data is carried in <c>functionResponse.parts[].inlineData</c>;
+    /// the envelope keeps the required <c>response</c> field present and gives the model a short hint.
+    /// </summary>
+    private static readonly object s_imageFunctionResponseEnvelope = new { status = "success", message = "Image data attached" };
+
     [JsonPropertyName("contents")]
     public IList<GeminiContent> Contents { get; set; } = null!;
 
@@ -194,14 +201,24 @@ private static List<GeminiPart> CreateGeminiParts(ChatMessageContent content)
             case GeminiChatMessageContent { CalledToolResults: not null } contentWithCalledTools:
                 // Add all function responses as separate parts in a single message
                 parts.AddRange(contentWithCalledTools.CalledToolResults.Select(toolResult =>
-                    new GeminiPart
+                {
+                    var resultValue = toolResult.FunctionResult.GetValue<object>();
+
+                    // Handle ImageContent for multimodal tool results (Gemini 3+ only)
+                    if (resultValue is ImageContent imageContent)
+                    {
+                        return CreateImageFunctionResponsePart(toolResult.FullyQualifiedName, imageContent);
+                    }
+
+                    return new GeminiPart
                     {
                         FunctionResponse = new GeminiPart.FunctionResponsePart
                         {
                             FunctionName = toolResult.FullyQualifiedName,
-                            Response = new(toolResult.FunctionResult.GetValue<object>())
+                            Response = new(resultValue)
                         }
-                    }));
+                    };
+                }));
                 break;
             case GeminiChatMessageContent { ToolCalls: not null } contentWithToolCalls:
                 parts.AddRange(contentWithToolCalls.ToolCalls.Select(toolCall =>
@@ -302,6 +319,37 @@ private static string GetMimeTypeFromImageContent(ImageContent imageContent)
                ?? throw new InvalidOperationException("Image content MimeType is empty.");
     }
 
+    /// <summary>
+    /// Creates a GeminiPart with FunctionResponse containing multimodal image data (Gemini 3+ only).
+    /// </summary>
+    private static GeminiPart CreateImageFunctionResponsePart(string functionName, ImageContent imageContent)
+    {
+        if (imageContent.Data is not { IsEmpty: false })
+        {
+            throw new InvalidOperationException("ImageContent in function result must contain binary data.");
+        }
+
+        return new GeminiPart
+        {
+            FunctionResponse = new GeminiPart.FunctionResponsePart
+            {
+                FunctionName = functionName,
+                Response = new(s_imageFunctionResponseEnvelope),
+                Parts =
+                [
+                    new GeminiPart.FunctionResponsePart.FunctionResponsePartContent
+                    {
+                        InlineData = new GeminiPart.InlineDataPart
+                        {
+                            MimeType = GetMimeTypeFromImageContent(imageContent),
+                            InlineData = Convert.ToBase64String(imageContent.Data.Value.ToArray())
+                        }
+                    }
+                ]
+            }
+        };
+    }
+
     private static GeminiPart CreateGeminiPartFromAudio(AudioContent audioContent)
     {
         // Binary data takes precedence over URI.