From 06fcca859213a4aba0d1c48d93ff6927647c60d2 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Fri, 15 Aug 2025 10:48:50 -0400 Subject: [PATCH] Update to M.E.AI 9.8 - Set new ChatMessage.CreatedAt property - Set and use new DataContent.Name property - Added citations to BedrockChatClient - Added implementation of IImageGenerator (as [Experimental] because the interface is itself [Experimental]) --- ...xtensions.Bedrock.MEAI.NetFramework.csproj | 2 +- ...Extensions.Bedrock.MEAI.NetStandard.csproj | 2 +- .../AWSSDK.Extensions.Bedrock.MEAI.nuspec | 6 +- .../AmazonBedrockRuntimeExtensions.cs | 15 ++ .../BedrockChatClient.cs | 39 +++- .../BedrockImageGenerator.cs | 197 ++++++++++++++++++ .../ExperimentalAttribute.cs | 33 +++ .../BedrockMEAITests.NetFramework.csproj | 2 +- 8 files changed, 287 insertions(+), 9 deletions(-) create mode 100644 extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockImageGenerator.cs create mode 100644 extensions/src/AWSSDK.Extensions.Bedrock.MEAI/ExperimentalAttribute.cs diff --git a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AWSSDK.Extensions.Bedrock.MEAI.NetFramework.csproj b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AWSSDK.Extensions.Bedrock.MEAI.NetFramework.csproj index 0456f29b866d..d28b6eed0b93 100644 --- a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AWSSDK.Extensions.Bedrock.MEAI.NetFramework.csproj +++ b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AWSSDK.Extensions.Bedrock.MEAI.NetFramework.csproj @@ -37,7 +37,7 @@ - + diff --git a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AWSSDK.Extensions.Bedrock.MEAI.NetStandard.csproj b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AWSSDK.Extensions.Bedrock.MEAI.NetStandard.csproj index f551ef2ff12c..98b381264471 100644 --- a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AWSSDK.Extensions.Bedrock.MEAI.NetStandard.csproj +++ b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AWSSDK.Extensions.Bedrock.MEAI.NetStandard.csproj @@ -41,7 +41,7 @@ - + diff --git a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AWSSDK.Extensions.Bedrock.MEAI.nuspec b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AWSSDK.Extensions.Bedrock.MEAI.nuspec index 5d8506d2ebc4..1aa7c758dc76 100644 --- a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AWSSDK.Extensions.Bedrock.MEAI.nuspec +++ b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AWSSDK.Extensions.Bedrock.MEAI.nuspec @@ -15,17 +15,17 @@ - + - + - + diff --git a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AmazonBedrockRuntimeExtensions.cs b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AmazonBedrockRuntimeExtensions.cs index 4a156803f6cb..836d5ad29fcd 100644 --- a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AmazonBedrockRuntimeExtensions.cs +++ b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/AmazonBedrockRuntimeExtensions.cs @@ -15,6 +15,7 @@ using Microsoft.Extensions.AI; using System; +using System.Diagnostics.CodeAnalysis; namespace Amazon.BedrockRuntime; @@ -53,4 +54,18 @@ public static IEmbeddingGenerator> AsIEmbeddingGenerato this IAmazonBedrockRuntime runtime, string? defaultModelId = null, int? defaultModelDimensions = null) => runtime is not null ? new BedrockEmbeddingGenerator(runtime, defaultModelId, defaultModelDimensions) : throw new ArgumentNullException(nameof(runtime)); + + /// Gets an for the specified instance. + /// The runtime instance to be represented as an . + /// + /// The default model ID to use when no model is specified in a request. If not specified, + /// a model must be provided in the passed to . + /// + /// An instance representing the instance. + /// is . + [Experimental("MEAI001")] + public static IImageGenerator AsIImageGenerator( + this IAmazonBedrockRuntime runtime, string? defaultModelId = null) => + runtime is not null ? new BedrockImageGenerator(runtime, defaultModelId) : + throw new ArgumentNullException(nameof(runtime)); } diff --git a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs index e20d746e6553..3f17d403ef25 100644 --- a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs +++ b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs @@ -83,6 +83,7 @@ public async Task GetResponseAsync( ChatMessage result = new() { + CreatedAt = DateTimeOffset.UtcNow, RawRepresentation = response.Output?.Message, Role = ChatRole.Assistant, MessageId = Guid.NewGuid().ToString("N"), @@ -97,6 +98,21 @@ public async Task GetResponseAsync( result.Contents.Add(new TextContent(text) { RawRepresentation = content }); } + if (content.CitationsContent is { } citations) + { + int count = Math.Min(citations.Citations?.Count ?? 0, citations.Content?.Count ?? 0); + for (int i = 0; i < count; i++) + { + TextContent tc = new(citations.Content![i]?.Text) { RawRepresentation = citations.Content![i] }; + tc.Annotations = [new CitationAnnotation() + { + Title = citations.Citations![i].Title, + Snippet = citations.Citations![i].SourceContent?.Select(c => c.Text).FirstOrDefault(), + }]; + result.Contents.Add(tc); + } + } + if (content.ReasoningContent is { ReasoningText.Text: not null } reasoningContent) { TextReasoningContent trc = new(reasoningContent.ReasoningText.Text) { RawRepresentation = content }; @@ -126,7 +142,11 @@ public async Task GetResponseAsync( if (content.Document is { Source.Bytes: { } documentBytes, Format: { } documentFormat }) { - result.Contents.Add(new DataContent(documentBytes.ToArray(), GetMimeType(documentFormat)) { RawRepresentation = content }); + result.Contents.Add(new DataContent(documentBytes.ToArray(), GetMimeType(documentFormat)) + { + RawRepresentation = content, + Name = content.Document.Name + }); } if (content.ToolUse is { } toolUse) @@ -143,7 +163,7 @@ public async Task GetResponseAsync( return new(result) { - CreatedAt = DateTimeOffset.UtcNow, + CreatedAt = result.CreatedAt, FinishReason = response.StopReason is not null ? GetChatFinishReason(response.StopReason) : null, RawRepresentation = response, ResponseId = Guid.NewGuid().ToString("N"), @@ -205,7 +225,7 @@ public async IAsyncEnumerable GetStreamingResponseAsync( if (contentBlockDelta.Delta.Text is string text) { - yield return new(ChatRole.Assistant, text) + ChatResponseUpdate textUpdate = new(ChatRole.Assistant, text) { CreatedAt = DateTimeOffset.UtcNow, MessageId = messageId, @@ -213,6 +233,18 @@ public async IAsyncEnumerable GetStreamingResponseAsync( FinishReason = finishReason, ResponseId = responseId, }; + + if (contentBlockDelta.Delta.Citation is { } citation && + (citation.Title is not null || citation.SourceContent is { Count: > 0 })) + { + textUpdate.Contents[0].Annotations = [new CitationAnnotation() + { + Title = citation.Title, + Snippet = citation.SourceContent?.Select(c => c.Text).FirstOrDefault(), + }]; + } + + yield return textUpdate; } if (contentBlockDelta.Delta.ReasoningContent is { Text: not null } reasoningContent) @@ -468,6 +500,7 @@ private static List CreateContents(ChatMessage message) { Source = new() { Bytes = new(dc.Data.ToArray()) }, Format = docFormat, + Name = dc.Name ?? "file", } }); } diff --git a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockImageGenerator.cs b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockImageGenerator.cs new file mode 100644 index 000000000000..fe61f1ac8854 --- /dev/null +++ b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockImageGenerator.cs @@ -0,0 +1,197 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +using Amazon.BedrockRuntime.Model; +using Microsoft.Extensions.AI; +using System; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; + +namespace Amazon.BedrockRuntime; + +[Experimental("MEAI001")] +internal sealed partial class BedrockImageGenerator : IImageGenerator +{ + /// The wrapped instance. + private readonly IAmazonBedrockRuntime _runtime; + /// Default model ID to use when no model is specified in the request. + private readonly string? _modelId; + /// Metadata describing the image generator. + private readonly ImageGeneratorMetadata _metadata; + + /// + /// Initializes a new instance of the class. + /// + /// The instance to wrap. + /// Model ID to use as the default when no model ID is specified in a request. + public BedrockImageGenerator(IAmazonBedrockRuntime runtime, string? defaultModelId) + { + Debug.Assert(runtime is not null); + + _runtime = runtime!; + _modelId = defaultModelId; + + _metadata = new(AmazonBedrockRuntimeExtensions.ProviderName, defaultModelId: defaultModelId); + } + + public void Dispose() + { + // Do not dispose of _runtime, as this instance doesn't own it. + } + + /// + + /// + public object? GetService(Type serviceType, object? serviceKey) + { + if (serviceType is null) + { + throw new ArgumentNullException(nameof(serviceType)); + } + + return + serviceKey is not null ? null : + serviceType == typeof(ImageGeneratorMetadata) ? _metadata : + serviceType.IsInstanceOfType(_runtime) ? _runtime : + serviceType.IsInstanceOfType(this) ? this : + null; + } + + public async Task GenerateAsync( + ImageGenerationRequest request, ImageGenerationOptions? options = null, CancellationToken cancellationToken = default) + { + if (request is null) + { + throw new ArgumentNullException(nameof(request)); + } + + int numImages = options?.Count ?? 1; + if (numImages < 1) + { + throw new ArgumentOutOfRangeException(nameof(options), "The number of images must be at least 1."); + } + + InvokeModelRequest invokeRequest = options?.RawRepresentationFactory?.Invoke(this) as InvokeModelRequest ?? new(); + invokeRequest.ModelId ??= options?.ModelId ?? _modelId; + invokeRequest.Accept ??= "application/json"; + invokeRequest.ContentType ??= "application/json"; + if (invokeRequest.Body is null) + { + JsonObject body = new(); + + // Each model has its own way of specifying the prompt and image generation parameters, unfortunately. + // The following logic handles the most common cases today, but may need to be extended for + // future models. + + if (invokeRequest.ModelId?.IndexOf("stability", StringComparison.OrdinalIgnoreCase) >= 0) + { + // Stability AI models + + if (invokeRequest.ModelId?.IndexOf("stable-diffusion", StringComparison.OrdinalIgnoreCase) >= 0) + { + JsonArray textPrompts = new(); + for (int i = 0; i < numImages; i++) + { + textPrompts.Add((JsonNode)new JsonObject { ["text"] = request.Prompt ?? "" }); + } + body["text_prompts"] = textPrompts; + + if (options?.ImageSize?.Width is int width && options.ImageSize?.Height is int height) + { + body["width"] = width; + body["height"] = height; + } + } + else + { + body["prompt"] = request.Prompt ?? ""; + } + } + else + { + // Amazon models (e.g. Titan, Nova Canvas) + + body["taskType"] = "TEXT_IMAGE"; + body["textToImageParams"] = new JsonObject { ["text"] = request.Prompt ?? "" }; + + JsonObject imageGenerationConfig = new() + { + ["seed"] = +#if NET + Random.Shared.Next(), +#else + new Random().Next(), +#endif + }; + + if (options?.ImageSize?.Width is int width && options.ImageSize?.Height is int height) + { + imageGenerationConfig["width"] = width; + imageGenerationConfig["height"] = height; + } + + if (numImages > 1) + { + imageGenerationConfig["numberOfImages"] = numImages; + } + + body["imageGenerationConfig"] = imageGenerationConfig; + } + + invokeRequest.Body = new MemoryStream(JsonSerializer.SerializeToUtf8Bytes(body, BedrockJsonContext.Default.JsonNode)); + } + + InvokeModelResponse rawResponse = await _runtime.InvokeModelAsync(invokeRequest, cancellationToken).ConfigureAwait(false); + + ImageGenerationResponse result = new() { RawRepresentation = rawResponse }; + + using JsonDocument doc = JsonDocument.Parse(rawResponse.Body); + JsonElement root = doc.RootElement; + + if (root.TryGetProperty("artifacts", out JsonElement artifactElement) && artifactElement.ValueKind == JsonValueKind.Array) + { + foreach (var element in artifactElement.EnumerateArray()) + { + if (element.TryGetProperty("base64", out JsonElement base64Element) && + base64Element.ValueKind == JsonValueKind.String) + { + result.Contents.Add(new DataContent(Convert.FromBase64String(base64Element.GetString()!), "image/png")); + } + } + } + else if (root.TryGetProperty("images", out JsonElement imagesElement) && imagesElement.ValueKind == JsonValueKind.Array) + { + foreach (var image in imagesElement.EnumerateArray()) + { + if (image.ValueKind == JsonValueKind.String) + { + result.Contents.Add(new DataContent(Convert.FromBase64String(image.GetString()!), "image/png")); + } + } + } + + if (result.Contents is not { Count: > 0 }) + { + throw new InvalidOperationException("Image generation did not produce any images."); + } + + return result; + } +} \ No newline at end of file diff --git a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/ExperimentalAttribute.cs b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/ExperimentalAttribute.cs new file mode 100644 index 000000000000..9aac6ad91e8e --- /dev/null +++ b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/ExperimentalAttribute.cs @@ -0,0 +1,33 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#if !NET +namespace System.Diagnostics.CodeAnalysis; + +// Polyfill for [Experimental] + +[AttributeUsage( + AttributeTargets.Assembly | AttributeTargets.Module | AttributeTargets.Class | AttributeTargets.Struct | + AttributeTargets.Enum | AttributeTargets.Constructor | AttributeTargets.Method | AttributeTargets.Property | + AttributeTargets.Field | AttributeTargets.Event | AttributeTargets.Interface | AttributeTargets.Delegate, + Inherited = false)] +internal sealed class ExperimentalAttribute : Attribute +{ + public ExperimentalAttribute(string diagnosticId) => DiagnosticId = diagnosticId; + public string DiagnosticId { get; } + public string? Message { get; set; } + public string? UrlFormat { get; set; } +} +#endif \ No newline at end of file diff --git a/extensions/test/BedrockMEAITests/BedrockMEAITests.NetFramework.csproj b/extensions/test/BedrockMEAITests/BedrockMEAITests.NetFramework.csproj index fefa9ec7170a..3c5d445d8278 100644 --- a/extensions/test/BedrockMEAITests/BedrockMEAITests.NetFramework.csproj +++ b/extensions/test/BedrockMEAITests/BedrockMEAITests.NetFramework.csproj @@ -18,7 +18,7 @@ - +