feat(api): realtime API updates

Author: stainless-app[bot]

.stats.yml

@@ -1,4 +1,4 @@
-configured_endpoints: 119
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-8517ffa1004e31ca2523d617629e64be6fe4f13403ddfd9db5b3be002656cbde.yml
-openapi_spec_hash: b64dd8c8b23082a7aa2a3e5c5fffd8bd
-config_hash: fe0ea26680ac2075a6cd66416aefe7db
+configured_endpoints: 118
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-356b4364203ff36d7724074cd04f6e684253bfcc3c9d969122d730aa7bc51b46.yml
+openapi_spec_hash: 4ab8e96f52699bc3d2b0c4432aa92af8
+config_hash: b854932c0ea24b400bdd64e4376936bd

README.md

@@ -226,7 +226,7 @@ async def main():
 asyncio.run(main())
 ```
 
-## Realtime API beta
+## Realtime API
 
 The Realtime API enables you to build low-latency, multi-modal conversational experiences. It currently supports text and audio as both input and output, as well as [function calling](https://platform.openai.com/docs/guides/function-calling) through a WebSocket connection.
 
@@ -243,7 +243,7 @@ from openai import AsyncOpenAI
 async def main():
     client = AsyncOpenAI()
 
-    async with client.beta.realtime.connect(model="gpt-4o-realtime-preview") as connection:
+    async with client.realtime.connect(model="gpt-realtime") as connection:
         await connection.session.update(session={'modalities': ['text']})
 
         await connection.conversation.item.create(
@@ -277,7 +277,7 @@ Whenever an error occurs, the Realtime API will send an [`error` event](https://
 ```py
 client = AsyncOpenAI()
 
-async with client.beta.realtime.connect(model="gpt-4o-realtime-preview") as connection:
+async with client.realtime.connect(model="gpt-realtime") as connection:
     ...
     async for event in connection:
         if event.type == 'error':

api.md

@@ -431,6 +431,7 @@ from openai.types.webhooks import (
     FineTuningJobCancelledWebhookEvent,
     FineTuningJobFailedWebhookEvent,
     FineTuningJobSucceededWebhookEvent,
+    RealtimeCallIncomingWebhookEvent,
     ResponseCancelledWebhookEvent,
     ResponseCompletedWebhookEvent,
     ResponseFailedWebhookEvent,
@@ -832,6 +833,7 @@ from openai.types.responses import (
     ToolChoiceMcp,
     ToolChoiceOptions,
     ToolChoiceTypes,
+    WebSearchPreviewTool,
     WebSearchTool,
 )
 ```
@@ -855,6 +857,115 @@ Methods:
 
 - <code title="get /responses/{response_id}/input_items">client.responses.input_items.<a href="./src/openai/resources/responses/input_items.py">list</a>(response_id, \*\*<a href="src/openai/types/responses/input_item_list_params.py">params</a>) -> <a href="./src/openai/types/responses/response_item.py">SyncCursorPage[ResponseItem]</a></code>
 
+# Realtime
+
+Types:
+
+```python
+from openai.types.realtime import (
+    ConversationCreatedEvent,
+    ConversationItem,
+    ConversationItemAdded,
+    ConversationItemCreateEvent,
+    ConversationItemCreatedEvent,
+    ConversationItemDeleteEvent,
+    ConversationItemDeletedEvent,
+    ConversationItemDone,
+    ConversationItemInputAudioTranscriptionCompletedEvent,
+    ConversationItemInputAudioTranscriptionDeltaEvent,
+    ConversationItemInputAudioTranscriptionFailedEvent,
+    ConversationItemInputAudioTranscriptionSegment,
+    ConversationItemRetrieveEvent,
+    ConversationItemTruncateEvent,
+    ConversationItemTruncatedEvent,
+    ConversationItemWithReference,
+    InputAudioBufferAppendEvent,
+    InputAudioBufferClearEvent,
+    InputAudioBufferClearedEvent,
+    InputAudioBufferCommitEvent,
+    InputAudioBufferCommittedEvent,
+    InputAudioBufferSpeechStartedEvent,
+    InputAudioBufferSpeechStoppedEvent,
+    InputAudioBufferTimeoutTriggered,
+    LogProbProperties,
+    McpListToolsCompleted,
+    McpListToolsFailed,
+    McpListToolsInProgress,
+    OutputAudioBufferClearEvent,
+    RateLimitsUpdatedEvent,
+    RealtimeAudioConfig,
+    RealtimeClientEvent,
+    RealtimeClientSecretConfig,
+    RealtimeConversationItemAssistantMessage,
+    RealtimeConversationItemFunctionCall,
+    RealtimeConversationItemFunctionCallOutput,
+    RealtimeConversationItemSystemMessage,
+    RealtimeConversationItemUserMessage,
+    RealtimeError,
+    RealtimeErrorEvent,
+    RealtimeMcpApprovalRequest,
+    RealtimeMcpApprovalResponse,
+    RealtimeMcpListTools,
+    RealtimeMcpProtocolError,
+    RealtimeMcpToolCall,
+    RealtimeMcpToolExecutionError,
+    RealtimeMcphttpError,
+    RealtimeResponse,
+    RealtimeResponseStatus,
+    RealtimeResponseUsage,
+    RealtimeResponseUsageInputTokenDetails,
+    RealtimeResponseUsageOutputTokenDetails,
+    RealtimeServerEvent,
+    RealtimeSession,
+    RealtimeSessionCreateRequest,
+    RealtimeToolChoiceConfig,
+    RealtimeToolsConfig,
+    RealtimeToolsConfigUnion,
+    RealtimeTracingConfig,
+    RealtimeTranscriptionSessionCreateRequest,
+    RealtimeTruncation,
+    ResponseAudioDeltaEvent,
+    ResponseAudioDoneEvent,
+    ResponseAudioTranscriptDeltaEvent,
+    ResponseAudioTranscriptDoneEvent,
+    ResponseCancelEvent,
+    ResponseContentPartAddedEvent,
+    ResponseContentPartDoneEvent,
+    ResponseCreateEvent,
+    ResponseCreatedEvent,
+    ResponseDoneEvent,
+    ResponseFunctionCallArgumentsDeltaEvent,
+    ResponseFunctionCallArgumentsDoneEvent,
+    ResponseMcpCallArgumentsDelta,
+    ResponseMcpCallArgumentsDone,
+    ResponseMcpCallCompleted,
+    ResponseMcpCallFailed,
+    ResponseMcpCallInProgress,
+    ResponseOutputItemAddedEvent,
+    ResponseOutputItemDoneEvent,
+    ResponseTextDeltaEvent,
+    ResponseTextDoneEvent,
+    SessionCreatedEvent,
+    SessionUpdateEvent,
+    SessionUpdatedEvent,
+    TranscriptionSessionCreated,
+    TranscriptionSessionUpdate,
+    TranscriptionSessionUpdatedEvent,
+)
+```
+
+## ClientSecrets
+
+Types:
+
+```python
+from openai.types.realtime import RealtimeSessionCreateResponse, ClientSecretCreateResponse
+```
+
+Methods:
+
+- <code title="post /realtime/client_secrets">client.realtime.client_secrets.<a href="./src/openai/resources/realtime/client_secrets.py">create</a>(\*\*<a href="src/openai/types/realtime/client_secret_create_params.py">params</a>) -> <a href="./src/openai/types/realtime/client_secret_create_response.py">ClientSecretCreateResponse</a></code>
+
 # Conversations
 
 Types:

examples/realtime/audio_util.py

@@ -11,7 +11,7 @@
 import sounddevice as sd
 from pydub import AudioSegment
 
-from openai.resources.beta.realtime.realtime import AsyncRealtimeConnection
+from openai.resources.realtime.realtime import AsyncRealtimeConnection
 
 CHUNK_LENGTH_S = 0.05  # 100ms
 SAMPLE_RATE = 24000

examples/realtime/azure_realtime.py

@@ -26,10 +26,16 @@ async def main() -> None:
         azure_ad_token_provider=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default"),
         api_version="2024-10-01-preview",
     )
-    async with client.beta.realtime.connect(
-        model="gpt-4o-realtime-preview",  # deployment name for your model
+    async with client.realtime.connect(
+        model="gpt-realtime",  # deployment name for your model
     ) as connection:
-        await connection.session.update(session={"modalities": ["text"]})  # type: ignore
+        await connection.session.update(
+            session={
+                "output_modalities": ["text"],
+                "model": "gpt-realtime",
+                "type": "realtime",
+            }
+        )
         while True:
             user_input = input("Enter a message: ")
             if user_input == "q":
@@ -44,9 +50,9 @@ async def main() -> None:
             )
             await connection.response.create()
             async for event in connection:
-                if event.type == "response.text.delta":
+                if event.type == "response.output_text.delta":
                     print(event.delta, flush=True, end="")
-                elif event.type == "response.text.done":
+                elif event.type == "response.output_text.done":
                     print()
                 elif event.type == "response.done":
                     break

examples/realtime/push_to_talk_app.py

@@ -38,8 +38,8 @@
 from textual.containers import Container
 
 from openai import AsyncOpenAI
-from openai.types.beta.realtime.session import Session
-from openai.resources.beta.realtime.realtime import AsyncRealtimeConnection
+from openai.types.realtime.session import Session
+from openai.resources.realtime.realtime import AsyncRealtimeConnection
 
 
 class SessionDisplay(Static):
@@ -154,13 +154,21 @@ async def on_mount(self) -> None:
         self.run_worker(self.send_mic_audio())
 
     async def handle_realtime_connection(self) -> None:
-        async with self.client.beta.realtime.connect(model="gpt-4o-realtime-preview") as conn:
+        async with self.client.realtime.connect(model="gpt-realtime") as conn:
             self.connection = conn
             self.connected.set()
 
             # note: this is the default and can be omitted
             # if you want to manually handle VAD yourself, then set `'turn_detection': None`
-            await conn.session.update(session={"turn_detection": {"type": "server_vad"}})
+            await conn.session.update(
+                session={
+                    "audio": {
+                        "input": {"turn_detection": {"type": "server_vad"}},
+                    },
+                    "model": "gpt-realtime",
+                    "type": "realtime",
+                }
+            )
 
             acc_items: dict[str, Any] = {}
 
@@ -176,7 +184,7 @@ async def handle_realtime_connection(self) -> None:
                     self.session = event.session
                     continue
 
-                if event.type == "response.audio.delta":
+                if event.type == "response.output_audio.delta":
                     if event.item_id != self.last_audio_item_id:
                         self.audio_player.reset_frame_count()
                         self.last_audio_item_id = event.item_id
@@ -185,7 +193,7 @@ async def handle_realtime_connection(self) -> None:
                     self.audio_player.add_data(bytes_data)
                     continue
 
-                if event.type == "response.audio_transcript.delta":
+                if event.type == "response.output_audio_transcript.delta":
                     try:
                         text = acc_items[event.item_id]
                     except KeyError:

examples/realtime/realtime.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env rye run python
+import asyncio
+
+from openai import AsyncOpenAI
+
+# Azure OpenAI Realtime Docs
+
+# How-to: https://learn.microsoft.com/azure/ai-services/openai/how-to/realtime-audio
+# Supported models and API versions: https://learn.microsoft.com/azure/ai-services/openai/how-to/realtime-audio#supported-models
+# Entra ID auth: https://learn.microsoft.com/azure/ai-services/openai/how-to/managed-identity
+
+
+async def main() -> None:
+    """The following example demonstrates how to configure OpenAI to use the Realtime API.
+    For an audio example, see push_to_talk_app.py and update the client and model parameter accordingly.
+
+    When prompted for user input, type a message and hit enter to send it to the model.
+    Enter "q" to quit the conversation.
+    """
+
+    client = AsyncOpenAI()
+    async with client.realtime.connect(
+        model="gpt-realtime",
+    ) as connection:
+        await connection.session.update(
+            session={
+                "output_modalities": ["text"],
+                "model": "gpt-realtime",
+                "type": "realtime",
+            }
+        )
+        while True:
+            user_input = input("Enter a message: ")
+            if user_input == "q":
+                break
+
+            await connection.conversation.item.create(
+                item={
+                    "type": "message",
+                    "role": "user",
+                    "content": [{"type": "input_text", "text": user_input}],
+                }
+            )
+            await connection.response.create()
+            async for event in connection:
+                if event.type == "response.output_text.delta":
+                    print(event.delta, flush=True, end="")
+                elif event.type == "response.output_text.done":
+                    print()
+                elif event.type == "response.done":
+                    break
+
+
+asyncio.run(main())

src/openai/__init__.py

@@ -379,6 +379,7 @@ def _reset_client() -> None:  # type: ignore[reportUnusedFunction]
     models as models,
     batches as batches,
     uploads as uploads,
+    realtime as realtime,
     webhooks as webhooks,
     responses as responses,
     containers as containers,