feat: heygen agora extension

ai_agents/.env.example

@@ -148,3 +148,6 @@ DUBVERSE_TTS_KEY=
 
 # Groq Cloud API Key
 GROQ_CLOUD_API_KEY=
+
+# Heygen Avatar API Key
+HEYGEN_API_KEY=

ai_agents/agents/examples/default/property.json

@@ -3,7 +3,7 @@
     "predefined_graphs": [
       {
         "name": "voice_assistant",
-        "auto_start": true,
+        "auto_start": false,
         "nodes": [
           {
             "type": "extension",
@@ -12,7 +12,7 @@
             "extension_group": "default",
             "property": {
               "app_id": "${env:AGORA_APP_ID}",
-              "app_certificate": "${env:AGORA_APP_CERTIFICATE|}",
+              "token": "<agora_token>",
               "channel": "ten_agent_test",
               "stream_id": 1234,
               "remote_stream_id": 123,
@@ -267,28 +267,28 @@
       },
       {
         "name": "voice_assistant_integrated_stt",
-        "auto_start": false,
+        "auto_start": true,
         "nodes": [
           {
             "type": "extension",
             "name": "agora_rtc",
             "addon": "agora_rtc",
             "extension_group": "default",
             "property": {
-              "enable_agora_asr": true,
-              "agora_asr_vendor_name": "microsoft",
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "<agora_token>",
               "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
               "subscribe_audio": true,
+              "publish_audio": true,
               "publish_data": true,
+              "enable_agora_asr": true,
+              "agora_asr_vendor_name": "microsoft",
               "agora_asr_language": "en-US",
-              "remote_stream_id": 123,
-              "agora_asr_vendor_region": "${env:AZURE_STT_REGION|}",
-              "publish_audio": true,
-              "app_certificate": "${env:AGORA_APP_CERTIFICATE|}",
-              "agora_asr_session_control_file_path": "session_control.conf",
-              "app_id": "${env:AGORA_APP_ID}",
               "agora_asr_vendor_key": "${env:AZURE_STT_KEY|}",
-              "stream_id": 1234
+              "agora_asr_vendor_region": "${env:AZURE_STT_REGION|}",
+              "agora_asr_session_control_file_path": "session_control.conf"
             }
           },
           {
@@ -343,6 +343,26 @@
             "property": {
               "api_key": "${env:WEATHERAPI_API_KEY|}"
             }
+          },
+          {
+            "type": "extension",
+            "name": "avatar",
+            "addon": "generic_video_python",
+            "extension_group": "default",
+            "property": {
+              "agora_appid": "${env:AGORA_APP_ID}",
+              "agora_appcert": "${env:AGORA_APP_CERTIFICATE}",
+              "agora_channel_name": "ten_agent_test",
+              "agora_avatar_uid": 12345,
+              "input_audio_sample_rate": 48000,
+              "avatar_id": "abc123",
+              "video_encoding": "H264",
+              "enable_string_uid": true,
+              "activity_idle_timeout": 60,
+              "generic_video_api_key": "${env:GENERIC_VIDEO_API_KEY}",
+              "start_endpoint": "http://oai.agora.io:8764/session/start",
+              "stop_endpoint": "http://oai.agora.io:8764/session/stop"
+            }
           }
         ],
         "connections": [
@@ -444,59 +464,59 @@
             ]
           },
           {
-            "extension": "tts",
+            "extension": "interrupt_detector",
             "cmd": [
               {
                 "name": "flush",
                 "dest": [
                   {
-                    "extension": "agora_rtc"
+                    "extension": "llm"
                   }
                 ]
               }
             ],
-            "audio_frame": [
+            "data": [
               {
-                "name": "pcm_frame",
+                "name": "text_data",
                 "dest": [
                   {
-                    "extension": "agora_rtc"
+                    "extension": "llm"
                   }
                 ]
               }
             ]
           },
           {
-            "extension": "interrupt_detector",
+            "extension": "weatherapi_tool_python",
             "cmd": [
               {
-                "name": "flush",
+                "name": "tool_register",
                 "dest": [
                   {
                     "extension": "llm"
                   }
                 ]
               }
-            ],
-            "data": [
+            ]
+          },
+          {
+            "extension": "tts",
+            "cmd": [
               {
-                "name": "text_data",
+                "name": "flush",
                 "dest": [
                   {
-                    "extension": "llm"
+                    "extension": "avatar"
                   }
                 ]
               }
-            ]
-          },
-          {
-            "extension": "weatherapi_tool_python",
-            "cmd": [
+            ],
+            "audio_frame": [
               {
-                "name": "tool_register",
+                "name": "pcm_frame",
                 "dest": [
                   {
-                    "extension": "llm"
+                    "extension": "avatar"
                   }
                 ]
               }
@@ -515,7 +535,7 @@
             "extension_group": "rtc",
             "property": {
               "app_id": "${env:AGORA_APP_ID}",
-              "app_certificate": "${env:AGORA_APP_CERTIFICATE|}",
+              "token": "",
               "channel": "ten_agent_test",
               "stream_id": 1234,
               "remote_stream_id": 123,
@@ -689,7 +709,7 @@
             "extension_group": "default",
             "property": {
               "app_id": "${env:AGORA_APP_ID}",
-              "app_certificate": "${env:AGORA_APP_CERTIFICATE|}",
+              "token": "<agora_token>",
               "channel": "ten_agent_test",
               "stream_id": 1234,
               "remote_stream_id": 123,
@@ -950,7 +970,7 @@
             "extension_group": "default",
             "property": {
               "app_id": "${env:AGORA_APP_ID}",
-              "app_certificate": "${env:AGORA_APP_CERTIFICATE|}",
+              "token": "<agora_token>",
               "channel": "ten_agent_test",
               "stream_id": 1234,
               "remote_stream_id": 123,
@@ -1191,7 +1211,7 @@
             "extension_group": "rtc",
             "property": {
               "app_id": "${env:AGORA_APP_ID}",
-              "app_certificate": "${env:AGORA_APP_CERTIFICATE|}",
+              "token": "",
               "channel": "ten_agent_test",
               "stream_id": 1234,
               "remote_stream_id": 123,

ai_agents/agents/ten_packages/extension/generic_video_python/README.md

@@ -0,0 +1,29 @@
+# generic_video_python
+
+<!-- brief introduction for the extension -->
+
+## Features
+
+<!-- main features introduction -->
+
+- xxx feature
+
+## API
+
+Refer to `api` definition in [manifest.json] and default values in [property.json](property.json).
+
+<!-- Additional API.md can be referred to if extra introduction needed -->
+
+## Development
+
+### Build
+
+<!-- build dependencies and steps -->
+
+### Unit test
+
+<!-- how to do unit test for the extension -->
+
+## Misc
+
+<!-- others if applicable -->

ai_agents/agents/ten_packages/extension/generic_video_python/__init__.py

@@ -0,0 +1,6 @@
+#
+# This file is part of TEN Framework, an open source project.
+# Licensed under the Apache License, Version 2.0.
+# See the LICENSE file for more information.
+#
+from . import addon

ai_agents/agents/ten_packages/extension/generic_video_python/addon.py

@@ -0,0 +1,18 @@
+#
+# This file is part of TEN Framework, an open source project.
+# Licensed under the Apache License, Version 2.0.
+# See the LICENSE file for more information.
+#
+from ten import (
+    Addon,
+    register_addon_as_extension,
+    TenEnv,
+)
+from .extension import GenericVideoExtension
+
+
+@register_addon_as_extension("generic_video_python")
+class GenericVideoExtensionAddon(Addon):
+    def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None:
+        ten_env.log_info("on_create_instance")
+        ten_env.on_create_instance_done(GenericVideoExtension(name), context)