Merge branch 'main' into feat/multimodel-vision

plutoless · web-flow · commit e3afe74cf95b · 2024-11-19T00:18:46.000+08:00
diff --git a/agents/property.json b/agents/property.json
@@ -4468,10 +4468,6 @@
                   {
                     "extension_group": "asr",
                     "extension": "deepgram_asr"
-                  },
-                  {
-                    "extension_group": "transcriber",
-                    "extension": "message_collector"
                   }
                 ]
               }
@@ -4716,10 +4712,6 @@
                   {
                     "extension_group": "asr",
                     "extension": "deepgram_asr"
-                  },
-                  {
-                    "extension_group": "transcriber",
-                    "extension": "message_collector"
                   }
                 ]
               }
diff --git a/agents/ten_packages/extension/deepgram_asr_python/manifest.json b/agents/ten_packages/extension/deepgram_asr_python/manifest.json
@@ -26,7 +26,8 @@
         },
         "audio_frame_in": [
             {
-                "name": "pcm_frame"
+                "name": "pcm_frame",
+                "property": {}
             }
         ],
         "cmd_in": [
diff --git a/agents/ten_packages/extension/message_collector_rtm/manifest.json b/agents/ten_packages/extension/message_collector_rtm/manifest.json
@@ -48,6 +48,18 @@
             "type": "string"
           }
         }
+      },
+      {
+        "name": "rtm_storage_event",
+        "property": {}
+      },
+      {
+        "name": "rtm_presence_event",
+        "property": {}
+      },
+      {
+        "name": "rtm_lock_event",
+        "property": {}
       }
     ],
     "data_out": [
@@ -63,6 +75,12 @@
         }
       }
     ],
+    "cmd_in": [
+      {
+        "name": "on_user_audio_track_state_changed",
+        "property": {}
+      }
+    ],
     "cmd_out": [
       {
         "name": "publish",
diff --git a/agents/ten_packages/extension/openai_chatgpt_python/manifest.json b/agents/ten_packages/extension/openai_chatgpt_python/manifest.json
@@ -83,15 +83,57 @@
         "name": "flush"
       },
       {
-        "name": "on_user_joined"
+        "name": "on_user_joined",
+        "property": {}
       },
       {
-        "name": "on_user_left"
+        "name": "on_user_left",
+        "property": {}
+      },
+      {
+        "name": "tool_register",
+        "property": {
+          "name": {
+            "type": "string"
+          },
+          "description": {
+            "type": "string"
+          },
+          "parameters": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "name",
+          "description",
+          "parameters"
+        ],
+        "result": {
+          "property": {
+            "response": {
+              "type": "string"
+            }
+          }
+        }
       }
     ],
     "cmd_out": [
       {
         "name": "flush"
+      },
+      {
+        "name": "tool_call",
+        "property": {
+          "name": {
+            "type": "string"
+          },
+          "args": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "name"
+        ]
       }
     ],
     "video_frame_in": [
diff --git a/agents/ten_packages/extension/transcribe_asr_python/manifest.json b/agents/ten_packages/extension/transcribe_asr_python/manifest.json
@@ -29,18 +29,22 @@
         },
         "audio_frame_in": [
             {
-                "name": "pcm_frame"
+                "name": "pcm_frame",
+                "property": {}
             }
         ],
         "cmd_in": [
             {
-                "name": "on_user_joined"
+                "name": "on_user_joined",
+                "property": {}
             },
             {
-                "name": "on_user_left"
+                "name": "on_user_left",
+                "property": {}
             },
             {
-                "name": "on_connection_failure"
+                "name": "on_connection_failure",
+                "property": {}
             }
         ],
         "data_out": [
diff --git a/agents/ten_packages/extension/vision_analyze_tool_python/manifest.json b/agents/ten_packages/extension/vision_analyze_tool_python/manifest.json
@@ -20,5 +20,57 @@
       "tests/**"
     ]
   },
-  "api": {}
+  "api": {
+    "property": {},
+    "cmd_in": [
+      {
+        "name": "tool_call",
+        "property": {
+          "name": {
+            "type": "string"
+          },
+          "args": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "name"
+        ]
+      }
+    ],
+    "cmd_out": [
+      {
+        "name": "tool_register",
+        "property": {
+          "name": {
+            "type": "string"
+          },
+          "description": {
+            "type": "string"
+          },
+          "parameters": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "name",
+          "description",
+          "parameters"
+        ],
+        "result": {
+          "property": {
+            "response": {
+              "type": "string"
+            }
+          }
+        }
+      }
+    ],
+    "video_frame_in": [
+      {
+        "name": "video_frame",
+        "property": {}
+      }
+    ]
+  }
 }
diff --git a/agents/ten_packages/extension/vision_tool_python/manifest.json b/agents/ten_packages/extension/vision_tool_python/manifest.json
@@ -20,5 +20,57 @@
       "tests/**"
     ]
   },
-  "api": {}
+  "api": {
+    "property": {},
+    "cmd_in": [
+      {
+        "name": "tool_call",
+        "property": {
+          "name": {
+            "type": "string"
+          },
+          "args": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "name"
+        ]
+      }
+    ],
+    "cmd_out": [
+      {
+        "name": "tool_register",
+        "property": {
+          "name": {
+            "type": "string"
+          },
+          "description": {
+            "type": "string"
+          },
+          "parameters": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "name",
+          "description",
+          "parameters"
+        ],
+        "result": {
+          "property": {
+            "response": {
+              "type": "string"
+            }
+          }
+        }
+      }
+    ],
+    "video_frame_in": [
+      {
+        "name": "video_frame",
+        "property": {}
+      }
+    ]
+  }
 }

Original file line number	Diff line number	Diff line change
`@@ -4468,10 +4468,6 @@`
`4468`	`4468`	`{`
`4469`	`4469`	`"extension_group": "asr",`
`4470`	`4470`	`"extension": "deepgram_asr"`
`4471`		`- },`
`4472`		`- {`
`4473`		`- "extension_group": "transcriber",`
`4474`		`- "extension": "message_collector"`
`4475`	`4471`	`}`
`4476`	`4472`	`]`
`4477`	`4473`	`}`
`@@ -4716,10 +4712,6 @@`
`4716`	`4712`	`{`
`4717`	`4713`	`"extension_group": "asr",`
`4718`	`4714`	`"extension": "deepgram_asr"`
`4719`		`- },`
`4720`		`- {`
`4721`		`- "extension_group": "transcriber",`
`4722`		`- "extension": "message_collector"`
`4723`	`4715`	`}`
`4724`	`4716`	`]`
`4725`	`4717`	`}`
Original file line number	Diff line number	Diff line change
`@@ -26,7 +26,8 @@`
`26`	`26`	`},`
`27`	`27`	`"audio_frame_in": [`
`28`	`28`	`{`
`29`		`- "name": "pcm_frame"`
	`29`	`+ "name": "pcm_frame",`
	`30`	`+ "property": {}`
`30`	`31`	`}`
`31`	`32`	`],`
`32`	`33`	`"cmd_in": [`
Original file line number	Diff line number	Diff line change
`@@ -48,6 +48,18 @@`
`48`	`48`	`"type": "string"`
`49`	`49`	`}`
`50`	`50`	`}`
	`51`	`+ },`
	`52`	`+ {`
	`53`	`+ "name": "rtm_storage_event",`
	`54`	`+ "property": {}`
	`55`	`+ },`
	`56`	`+ {`
	`57`	`+ "name": "rtm_presence_event",`
	`58`	`+ "property": {}`
	`59`	`+ },`
	`60`	`+ {`
	`61`	`+ "name": "rtm_lock_event",`
	`62`	`+ "property": {}`
`51`	`63`	`}`
`52`	`64`	`],`
`53`	`65`	`"data_out": [`
`@@ -63,6 +75,12 @@`
`63`	`75`	`}`
`64`	`76`	`}`
`65`	`77`	`],`
	`78`	`+ "cmd_in": [`
	`79`	`+ {`
	`80`	`+ "name": "on_user_audio_track_state_changed",`
	`81`	`+ "property": {}`
	`82`	`+ }`
	`83`	`+ ],`
`66`	`84`	`"cmd_out": [`
`67`	`85`	`{`
`68`	`86`	`"name": "publish",`
Original file line number	Diff line number	Diff line change
`@@ -29,18 +29,22 @@`
`29`	`29`	`},`
`30`	`30`	`"audio_frame_in": [`
`31`	`31`	`{`
`32`		`- "name": "pcm_frame"`
	`32`	`+ "name": "pcm_frame",`
	`33`	`+ "property": {}`
`33`	`34`	`}`
`34`	`35`	`],`
`35`	`36`	`"cmd_in": [`
`36`	`37`	`{`
`37`		`- "name": "on_user_joined"`
	`38`	`+ "name": "on_user_joined",`
	`39`	`+ "property": {}`
`38`	`40`	`},`
`39`	`41`	`{`
`40`		`- "name": "on_user_left"`
	`42`	`+ "name": "on_user_left",`
	`43`	`+ "property": {}`
`41`	`44`	`},`
`42`	`45`	`{`
`43`		`- "name": "on_connection_failure"`
	`46`	`+ "name": "on_connection_failure",`
	`47`	`+ "property": {}`
`44`	`48`	`}`
`45`	`49`	`],`
`46`	`50`	`"data_out": [`