.Net: Update to latest M.E.AI{.Abstractions} + OpenAI Related Packages

dotnet/Directory.Packages.props

@@ -19,7 +19,7 @@
     <PackageVersion Include="AWSSDK.SecurityToken" Version="4.0.1.3" />
     <PackageVersion Include="Azure.AI.Agents.Persistent" Version="1.0.0" />
     <PackageVersion Include="Azure.AI.ContentSafety" Version="1.0.0" />
-    <PackageVersion Include="Azure.AI.OpenAI" Version="[2.2.0-beta.4]" />
+    <PackageVersion Include="Azure.AI.OpenAI" Version="[2.2.0-beta.5]" />
     <PackageVersion Include="Azure.AI.Projects" Version="1.0.0-beta.9" />
     <PackageVersion Include="Azure.Identity" Version="1.14.1" />
     <PackageVersion Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.3.0" />
@@ -70,7 +70,7 @@
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="Npgsql" Version="8.0.7" />
     <PackageVersion Include="OllamaSharp" Version="5.2.3" />
-    <PackageVersion Include="OpenAI" Version="[2.2.0-beta.4]" />
+    <PackageVersion Include="OpenAI" Version="[2.2.0]" />
     <PackageVersion Include="OpenTelemetry.Exporter.Console" Version="1.12.0" />
     <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.12.0" />
     <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.12.0" />
@@ -88,7 +88,7 @@
     <PackageVersion Include="System.Memory.Data" Version="8.0.1" />
     <PackageVersion Include="System.Net.Http" Version="4.3.4" />
     <PackageVersion Include="System.Numerics.Tensors" Version="9.0.6" />
-    <PackageVersion Include="System.Text.Json" Version="8.0.5" />
+    <PackageVersion Include="System.Text.Json" Version="8.0.6" />
     <PackageVersion Include="System.ValueTuple" Version="4.6.1" />
     <PackageVersion Include="System.Threading.Tasks.Extensions" Version="4.6.3" />
     <PackageVersion Include="SharpA2A.Core" Version="0.2.1-preview.1" />
@@ -97,10 +97,10 @@
     <!-- Tokenizers -->
     <PackageVersion Include="Microsoft.ML.Tokenizers" Version="1.0.2" />
     <!-- Microsoft.Extensions.* -->
-    <PackageVersion Include="Microsoft.Extensions.AI" Version="9.6.0" />
-    <PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="9.5.0" />
-    <PackageVersion Include="Microsoft.Extensions.AI.AzureAIInference" Version="9.6.0-preview.1.25310.2" />
-    <PackageVersion Include="Microsoft.Extensions.AI.OpenAI" Version="9.5.0-preview.1.25265.7" />
+    <PackageVersion Include="Microsoft.Extensions.AI" Version="9.7.1" />
+    <PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="9.7.1" />
+    <PackageVersion Include="Microsoft.Extensions.AI.AzureAIInference" Version="9.7.1-preview.1.25365.4" />
+    <PackageVersion Include="Microsoft.Extensions.AI.OpenAI" Version="9.7.1-preview.1.25365.4" />
     <PackageVersion Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />

dotnet/samples/Demos/OpenAIRealtime/OpenAIRealtime.csproj

@@ -5,7 +5,7 @@
     <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
-    <NoWarn>$(NoWarn);VSTHRD111,CA2007,CS8618,CS1591,CA1052,SKEXP0001</NoWarn>
+    <NoWarn>$(NoWarn);VSTHRD111,CA2007,CS8618,CS1591,CA1052,CA1810,SKEXP0001</NoWarn>
     <UserSecretsId>5ee045b0-aea3-4f08-8d31-32d1a6f8fed0</UserSecretsId>
   </PropertyGroup>
 

dotnet/samples/Demos/OpenAIRealtime/Program.cs

@@ -8,15 +8,15 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.Connectors.OpenAI;
-using OpenAI.RealtimeConversation;
+using OpenAI.Realtime;
 
 namespace OpenAIRealtime;
 
 #pragma warning disable OPENAI002
 
 /// <summary>
 /// Demonstrates the use of the OpenAI Realtime API with function calling and Semantic Kernel.
-/// For conversational experiences, it is recommended to use <see cref="RealtimeConversationClient"/> from the Azure/OpenAI SDK.
+/// For conversational experiences, it is recommended to use <see cref="RealtimeClient"/> from the Azure/OpenAI SDK.
 /// Since the OpenAI Realtime API supports function calling, the example shows how to combine it with Semantic Kernel plugins and functions.
 /// </summary>
 internal sealed class Program
@@ -33,16 +33,16 @@ public static async Task Main(string[] args)
         kernel.ImportPluginFromType<WeatherPlugin>();
 
         // Start a new conversation session.
-        using RealtimeConversationSession session = await realtimeConversationClient.StartConversationSessionAsync();
+        using RealtimeSession session = await realtimeConversationClient.StartConversationSessionAsync("gpt-4o-realtime-preview");
 
         // Initialize session options.
         // Session options control connection-wide behavior shared across all conversations,
         // including audio input format and voice activity detection settings.
         ConversationSessionOptions sessionOptions = new()
         {
             Voice = ConversationVoice.Alloy,
-            InputAudioFormat = ConversationAudioFormat.Pcm16,
-            OutputAudioFormat = ConversationAudioFormat.Pcm16,
+            InputAudioFormat = RealtimeAudioFormat.Pcm16,
+            OutputAudioFormat = RealtimeAudioFormat.Pcm16,
             InputTranscriptionOptions = new()
             {
                 Model = "whisper-1",
@@ -62,13 +62,12 @@ public static async Task Main(string[] args)
         }
 
         // Configure session with defined options.
-        await session.ConfigureSessionAsync(sessionOptions);
+        await session.ConfigureConversationSessionAsync(sessionOptions);
 
         // Items such as user, assistant, or system messages, as well as input audio, can be sent to the session.
         // An example of sending user message to the session.
         // ConversationItem can be constructed from Microsoft.SemanticKernel.ChatMessageContent if needed by mapping the relevant fields.
-        await session.AddItemAsync(
-            ConversationItem.CreateUserMessage(["I'm trying to decide what to wear on my trip."]));
+        await session.AddItemAsync(RealtimeItem.CreateUserMessage(["I'm trying to decide what to wear on my trip."]));
 
         // Use audio file that contains a recorded question: "What's the weather like in San Francisco, California?"
         string inputAudioPath = FindFile("Assets\\realtime_whats_the_weather_pcm16_24khz_mono.wav");
@@ -82,7 +81,7 @@ await session.AddItemAsync(
         Dictionary<string, StringBuilder> functionArgumentBuildersById = [];
 
         // Define a loop to receive conversation updates in the session.
-        await foreach (ConversationUpdate update in session.ReceiveUpdatesAsync())
+        await foreach (RealtimeUpdate update in session.ReceiveUpdatesAsync())
         {
             // Notification indicating the start of the conversation session.
             if (update is ConversationSessionStartedUpdate sessionStartedUpdate)
@@ -92,21 +91,21 @@ await session.AddItemAsync(
             }
 
             // Notification indicating the start of detected voice activity.
-            if (update is ConversationInputSpeechStartedUpdate speechStartedUpdate)
+            if (update is InputAudioSpeechStartedUpdate speechStartedUpdate)
             {
                 Console.WriteLine(
                     $"  -- Voice activity detection started at {speechStartedUpdate.AudioStartTime}");
             }
 
             // Notification indicating the end of detected voice activity.
-            if (update is ConversationInputSpeechFinishedUpdate speechFinishedUpdate)
+            if (update is InputAudioSpeechFinishedUpdate speechFinishedUpdate)
             {
                 Console.WriteLine(
                     $"  -- Voice activity detection ended at {speechFinishedUpdate.AudioEndTime}");
             }
 
             // Notification indicating the start of item streaming, such as a function call or response message.
-            if (update is ConversationItemStreamingStartedUpdate itemStreamingStartedUpdate)
+            if (update is OutputStreamingStartedUpdate itemStreamingStartedUpdate)
             {
                 Console.WriteLine("  -- Begin streaming of new item");
                 if (!string.IsNullOrEmpty(itemStreamingStartedUpdate.FunctionName))
@@ -116,7 +115,7 @@ await session.AddItemAsync(
             }
 
             // Notification about item streaming delta, which may include audio transcript, audio bytes, or function arguments.
-            if (update is ConversationItemStreamingPartDeltaUpdate deltaUpdate)
+            if (update is OutputDeltaUpdate deltaUpdate)
             {
                 Console.Write(deltaUpdate.AudioTranscript);
                 Console.Write(deltaUpdate.Text);
@@ -148,7 +147,7 @@ await session.AddItemAsync(
 
             // Notification indicating the end of item streaming, such as a function call or response message.
             // At this point, audio transcript can be displayed on console, or a function can be called with aggregated arguments.
-            if (update is ConversationItemStreamingFinishedUpdate itemStreamingFinishedUpdate)
+            if (update is OutputStreamingFinishedUpdate itemStreamingFinishedUpdate)
             {
                 Console.WriteLine();
                 Console.WriteLine($"  -- Item streaming finished, item_id={itemStreamingFinishedUpdate.ItemId}");
@@ -176,7 +175,7 @@ await session.AddItemAsync(
                     var resultContent = await functionCallContent.InvokeAsync(kernel);
 
                     // Create a function call output conversation item with function call result.
-                    ConversationItem functionOutputItem = ConversationItem.CreateFunctionCallOutput(
+                    RealtimeItem functionOutputItem = RealtimeItem.CreateFunctionCallOutput(
                         callId: itemStreamingFinishedUpdate.FunctionCallId,
                         output: ProcessFunctionResult(resultContent.Result));
 
@@ -198,15 +197,15 @@ await session.AddItemAsync(
             }
 
             // Notification indicating the completion of transcription from input audio.
-            if (update is ConversationInputTranscriptionFinishedUpdate transcriptionCompletedUpdate)
+            if (update is InputAudioTranscriptionFinishedUpdate transcriptionCompletedUpdate)
             {
                 Console.WriteLine();
                 Console.WriteLine($"  -- User audio transcript: {transcriptionCompletedUpdate.Transcript}");
                 Console.WriteLine();
             }
 
             // Notification about completed model response turn.
-            if (update is ConversationResponseFinishedUpdate turnFinishedUpdate)
+            if (update is ResponseFinishedUpdate turnFinishedUpdate)
             {
                 Console.WriteLine($"  -- Model turn generation finished. Status: {turnFinishedUpdate.Status}");
 
@@ -226,7 +225,7 @@ await session.AddItemAsync(
             }
 
             // Notification about error in conversation session.
-            if (update is ConversationErrorUpdate errorUpdate)
+            if (update is RealtimeErrorUpdate errorUpdate)
             {
                 Console.WriteLine();
                 Console.WriteLine($"ERROR: {errorUpdate.Message}");
@@ -375,32 +374,30 @@ private static string FindFile(string fileName)
     }
 
     /// <summary>
-    /// Helper method to get an instance of <see cref="RealtimeConversationClient"/> based on provided
+    /// Helper method to get an instance of <see cref="RealtimeClient"/> based on provided
     /// OpenAI or Azure OpenAI configuration.
     /// </summary>
-    private static RealtimeConversationClient GetRealtimeConversationClient()
+    private static RealtimeClient GetRealtimeConversationClient()
     {
         var config = new ConfigurationBuilder()
             .AddUserSecrets<Program>()
             .AddEnvironmentVariables()
             .Build();
 
-        var openAIOptions = config.GetSection(OpenAIOptions.SectionName).Get<OpenAIOptions>();
-        var azureOpenAIOptions = config.GetSection(AzureOpenAIOptions.SectionName).Get<AzureOpenAIOptions>();
+        var openAIOptions = config.GetSection(OpenAIOptions.SectionName).Get<OpenAIOptions>()!;
+        var azureOpenAIOptions = config.GetSection(AzureOpenAIOptions.SectionName).Get<AzureOpenAIOptions>()!;
 
         if (openAIOptions is not null && openAIOptions.IsValid)
         {
-            return new RealtimeConversationClient(
-                model: "gpt-4o-realtime-preview",
-                credential: new ApiKeyCredential(openAIOptions.ApiKey));
+            return new RealtimeClient(new ApiKeyCredential(openAIOptions.ApiKey));
         }
         else if (azureOpenAIOptions is not null && azureOpenAIOptions.IsValid)
         {
             var client = new AzureOpenAIClient(
                 endpoint: new Uri(azureOpenAIOptions.Endpoint),
                 credential: new ApiKeyCredential(azureOpenAIOptions.ApiKey));
 
-            return client.GetRealtimeConversationClient(azureOpenAIOptions.DeploymentName);
+            return client.GetRealtimeClient();
         }
         else
         {

dotnet/src/Agents/OpenAI/Extensions/OpenAIResponseExtensions.cs

@@ -52,7 +52,7 @@ public static ChatMessageContent ToChatMessageContent(this OpenAIResponse respon
         }
         else if (item is ReasoningResponseItem reasoningResponseItem)
         {
-            if (reasoningResponseItem.SummaryTextParts is not null && reasoningResponseItem.SummaryTextParts.Count > 0)
+            if (reasoningResponseItem.SummaryParts is not null && reasoningResponseItem.SummaryParts.Count > 0)
             {
                 return new ChatMessageContent(AuthorRole.Assistant, item.ToChatMessageContentItemCollection(), innerContent: reasoningResponseItem);
             }
@@ -77,7 +77,7 @@ public static ChatMessageContentItemCollection ToChatMessageContentItemCollectio
         }
         else if (item is ReasoningResponseItem reasoningResponseItem)
         {
-            return reasoningResponseItem.SummaryTextParts.ToChatMessageContentItemCollection();
+            return reasoningResponseItem.SummaryParts.ToChatMessageContentItemCollection();
         }
         else if (item is FunctionCallResponseItem functionCallResponseItem)
         {
@@ -195,12 +195,15 @@ private static ChatMessageContentItemCollection ToChatMessageContentItemCollecti
         return collection;
     }
 
-    private static ChatMessageContentItemCollection ToChatMessageContentItemCollection(this IReadOnlyList<string> texts)
+    private static ChatMessageContentItemCollection ToChatMessageContentItemCollection(this IReadOnlyList<ReasoningSummaryPart> parts)
     {
         var collection = new ChatMessageContentItemCollection();
-        foreach (var text in texts)
+        foreach (var part in parts)
         {
-            collection.Add(new TextContent(text, innerContent: null));
+            if (part is ReasoningSummaryTextPart text)
+            {
+                collection.Add(new TextContent(text.Text, innerContent: text));
+            }
         }
         return collection;
     }

dotnet/src/Agents/UnitTests/Extensions/ResponseItemExtensionsTests.cs

@@ -66,7 +66,7 @@ public void VerifyToChatMessageContentFromInputFile()
     {
         // Arrange
         var fileBytes = new ReadOnlyMemory<byte>([1, 2, 3, 4, 5]);
-        IEnumerable<ResponseContentPart> contentParts = [ResponseContentPart.CreateInputFilePart("fileId", "fileName", new(fileBytes))];
+        IEnumerable<ResponseContentPart> contentParts = [ResponseContentPart.CreateInputFilePart(BinaryData.FromBytes(fileBytes), "text/plain", "fileName")];
         MessageResponseItem responseItem = ResponseItem.CreateUserMessageItem(contentParts);
 
         // Act
@@ -102,7 +102,7 @@ public void VerifyToChatMessageContentFromRefusal()
     public void VerifyToChatMessageContentFromReasoning()
     {
         // Arrange
-        IEnumerable<string> summaryParts = ["Foo"];
+        IEnumerable<ReasoningSummaryPart> summaryParts = [ReasoningSummaryPart.CreateTextPart("Foo")];
         ReasoningResponseItem responseItem = ResponseItem.CreateReasoningItem(summaryParts);
 
         // Act

dotnet/src/Agents/UnitTests/OpenAI/Extensions/OpenAIResponseExtensionsTests.cs

@@ -2,6 +2,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Reflection;
 using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.Agents.OpenAI;
@@ -178,46 +179,71 @@ private OpenAIResponse CreateMockOpenAIResponse(string model, IEnumerable<Respon
     private OpenAIResponse CreateMockOpenAIResponse(string id, DateTimeOffset createdAt, ResponseError error, string instructions, string model, string previousResponseId, float temperature, IEnumerable<ResponseTool> tools, float topP, IDictionary<string, string> metadata, ResponseIncompleteStatusDetails incompleteStatusDetails, IEnumerable<ResponseItem> outputItems, bool parallelToolCallsEnabled, ResponseToolChoice toolChoice)
     {
         Type type = typeof(OpenAIResponse);
+        var assembly = type.Assembly;
+        var internalServiceTierType = assembly.GetType("OpenAI.Internal.InternalServiceTier");
+        var nullableInternalServiceTierType = typeof(Nullable<>).MakeGenericType(internalServiceTierType!);
 
         ConstructorInfo? constructor = type.GetConstructor(
             BindingFlags.Instance | BindingFlags.NonPublic,
             null,
             [
+                typeof(IDictionary<string, string>),
+                typeof(float?),
+                typeof(float?),
+                nullableInternalServiceTierType,
                 typeof(string),
-                typeof(DateTimeOffset),
-                typeof(ResponseError),
+                typeof(bool?),
                 typeof(string),
+                typeof(IList<ResponseTool>),
                 typeof(string),
+                typeof(ResponseStatus?),
+                typeof(DateTimeOffset),
+                typeof(ResponseError),
+                typeof(ResponseTokenUsage),
                 typeof(string),
-                typeof(float),
-                typeof(IEnumerable<ResponseTool>),
-                typeof(float),
-                typeof(IDictionary<string, string>),
+                typeof(ResponseReasoningOptions),
+                typeof(int?),
+                typeof(ResponseTextOptions),
+                typeof(ResponseTruncationMode?),
                 typeof(ResponseIncompleteStatusDetails),
-                typeof(IEnumerable<ResponseItem>),
+                typeof(IList<ResponseItem>),
                 typeof(bool),
-                typeof(ResponseToolChoice)
+                typeof(ResponseToolChoice),
+                typeof(string),
+                typeof(string),
+                typeof(IDictionary<string, BinaryData>)
             ],
             null);
 
         if (constructor != null)
         {
             return (OpenAIResponse)constructor.Invoke(
                 [
-                    id,
-                    createdAt,
-                    error,
-                    instructions,
-                    model,
-                    previousResponseId,
-                    temperature,
-                    tools,
-                    topP,
                     metadata,
-                    incompleteStatusDetails,
-                    outputItems,
-                    parallelToolCallsEnabled,
-                    toolChoice
+                (float?)temperature,
+                (float?)topP,
+                null, // serviceTier
+                previousResponseId,
+                null, // background
+                instructions,
+                tools.ToList(),
+                id,
+                null, // status
+                createdAt,
+                error,
+                null, // usage
+                null, // endUserId
+                null, // reasoningOptions
+                null, // maxOutputTokenCount
+                null, // textOptions
+                null, // truncationMode
+                incompleteStatusDetails,
+                outputItems.ToList(),
+                parallelToolCallsEnabled,
+                toolChoice,
+                model,
+                "response",
+                null // additionalBinaryDataProperties
                 ]
             );
         }