Autogen models in flow

examples/nextjs-flow/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": ["next/core-web-vitals", "next/typescript"]
+}

examples/react-native-flow/app/index.tsx

@@ -19,7 +19,7 @@ import {
   useFlow,
   useFlowEventListener,
 } from '@speechmatics/flow-client-react';
-import { useCallback, useEffect, useState, useSyncExternalStore } from 'react';
+import { useCallback, useEffect, useState } from 'react';
 import { useSharedValue } from 'react-native-reanimated';
 import getFlowAPIJwt from '@/app/auth';
 import VolumeDisplay from '@/app/volume-display';

packages/flow-client-react/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@speechmatics/flow-client-react",
-  "version": "0.2.1",
+  "version": "1.0.0-rc.7.1",
   "description": "React hooks for interacting with the Speechmatics Flow API",
   "main": "dist/index.cjs",
   "module": "dist/index.js",

packages/flow-client-react/src/flow-provider.tsx

@@ -25,7 +25,7 @@ export function FlowProvider({
     if (data.message === 'Error') {
       console.error(data);
     } else if (data.message === 'ConversationStarted') {
-      setSessionId(data.asr_session_id);
+      setSessionId(data.id);
     }
   });
 

packages/flow-client-react/src/use-flow.ts

@@ -1,4 +1,4 @@
-import type { StartConversationMessage } from '@speechmatics/flow-client';
+import type { AddInput, StartConversation, ToolResult } from '@speechmatics/flow-client';
 import { useCallback, useContext, useMemo } from 'react';
 import { FlowContext } from './flow-context';
 
@@ -18,8 +18,8 @@ export function useFlow() {
         config,
         audioFormat,
       }: {
-        config: StartConversationMessage['conversation_config'];
-        audioFormat?: StartConversationMessage['audio_format'];
+        config: StartConversation['conversation_config'];
+        audioFormat?: StartConversation['audio_format'];
       },
     ) => client.startConversation(jwt, { config, audioFormat }),
     [client],
@@ -32,14 +32,27 @@ export function useFlow() {
     [client],
   );
 
+  const sendToolResult = useCallback(
+    (toolResult: Exclude<ToolResult, 'message'>) =>
+      client.sendToolResult(toolResult),
+    [client],
+  );
+
+  const sendInput = useCallback(
+    (input: Exclude<AddInput, 'message'>) => client.sendInput(input),
+    [client],
+  );
+
   return useMemo(
     () => ({
       startConversation,
       endConversation,
       sendAudio,
       socketState,
       sessionId,
+      sendToolResult,
+      sendInput,
     }),
-    [startConversation, endConversation, sendAudio, socketState, sessionId],
+    [startConversation, endConversation, sendAudio, socketState, sessionId, sendToolResult, sendInput],
   );
 }

packages/flow-client/models/AddInput.ts

@@ -0,0 +1,7 @@
+interface AddInput {
+  message: 'AddInput';
+  input: string;
+  interrupt_response?: boolean;
+  immediate?: boolean;
+}
+export type { AddInput };

packages/flow-client/models/AddPartialTranscript.ts

@@ -0,0 +1,9 @@
+import type { RecognitionMetadata } from './RecognitionMetadata';
+import type { RecognitionResult } from './RecognitionResult';
+interface AddPartialTranscript {
+  message: 'AddPartialTranscript';
+  format?: string;
+  metadata: RecognitionMetadata;
+  results: RecognitionResult[];
+}
+export type { AddPartialTranscript };

packages/flow-client/models/AddTranscript.ts

@@ -0,0 +1,9 @@
+import type { RecognitionMetadata } from './RecognitionMetadata';
+import type { RecognitionResult } from './RecognitionResult';
+interface AddTranscript {
+  message: 'AddTranscript';
+  format?: string;
+  metadata: RecognitionMetadata;
+  results: RecognitionResult[];
+}
+export type { AddTranscript };

packages/flow-client/models/AttachesToEnum.ts

@@ -0,0 +1,2 @@
+type AttachesToEnum = 'next' | 'previous' | 'none' | 'both';
+export type { AttachesToEnum };

packages/flow-client/models/AudioAdded.ts

@@ -0,0 +1,5 @@
+interface AudioAdded {
+  message: 'AudioAdded';
+  seq_no: number;
+}
+export type { AudioAdded };

packages/flow-client/models/AudioEnded.ts

@@ -0,0 +1,5 @@
+interface AudioEnded {
+  message: 'AudioEnded';
+  last_seq_no: number;
+}
+export type { AudioEnded };

packages/flow-client/models/AudioFormat.ts

@@ -0,0 +1,7 @@
+import type { AudioFormatRawEncodingEnum } from './AudioFormatRawEncodingEnum';
+interface AudioFormat {
+  type: 'raw';
+  encoding?: AudioFormatRawEncodingEnum;
+  sample_rate?: number;
+}
+export type { AudioFormat };

packages/flow-client/models/AudioFormatRawEncodingEnum.ts

@@ -0,0 +1,2 @@
+type AudioFormatRawEncodingEnum = 'pcm_f32le' | 'pcm_s16le' | 'mulaw';
+export type { AudioFormatRawEncodingEnum };

packages/flow-client/models/AudioReceived.ts

@@ -0,0 +1,5 @@
+interface AudioReceived {
+  message: 'AudioReceived';
+  seq_no: number;
+}
+export type { AudioReceived };

packages/flow-client/models/ConcurrentSessionUsageInfo.ts

@@ -0,0 +1,9 @@
+interface ConcurrentSessionUsageInfo {
+  reason: string;
+  message: 'Info';
+  usage: number;
+  quota: number;
+  last_udpated?: string;
+  type: 'concurrent_session_usage';
+}
+export type { ConcurrentSessionUsageInfo };

packages/flow-client/models/ConversationConfig.ts

@@ -0,0 +1,5 @@
+interface ConversationConfig {
+  template_id: string;
+  template_variables?: { [name: string]: string };
+}
+export type { ConversationConfig };

packages/flow-client/models/ConversationDurationLimitInfo.ts

@@ -0,0 +1,7 @@
+interface ConversationDurationLimitInfo {
+  reason: string;
+  message: 'Info';
+  conversation_duration_limit: number;
+  type: 'conversation_duration_limit';
+}
+export type { ConversationDurationLimitInfo };

packages/flow-client/models/ConversationEnded.ts

@@ -0,0 +1,4 @@
+interface ConversationEnded {
+  message: 'ConversationEnded';
+}
+export type { ConversationEnded };

packages/flow-client/models/ConversationEnding.ts

@@ -0,0 +1,4 @@
+interface ConversationEnding {
+  message: 'ConversationEnding';
+}
+export type { ConversationEnding };

packages/flow-client/models/ConversationStarted.ts

@@ -0,0 +1,8 @@
+import type { LanguagePackInfo } from './LanguagePackInfo';
+interface ConversationStarted {
+  message: 'ConversationStarted';
+  id?: string;
+  asr_session_id?: string;
+  language_pack_info?: LanguagePackInfo;
+}
+export type { ConversationStarted };

packages/flow-client/models/ConversationTerminationWarning.ts

@@ -0,0 +1,7 @@
+interface ConversationTerminationWarning {
+  reason: string;
+  message: 'Warning';
+  conversation_termination: number;
+  type: 'conversation_termination';
+}
+export type { ConversationTerminationWarning };

packages/flow-client/models/DebugObject.ts

@@ -0,0 +1,5 @@
+interface DebugObject {
+  llm?: boolean;
+  additionalProperties?: { [name: string]: unknown };
+}
+export type { DebugObject };

packages/flow-client/models/ErrorType.ts

@@ -0,0 +1,7 @@
+import type { ErrorTypeEnum } from './ErrorTypeEnum';
+interface ErrorType {
+  message: 'Error';
+  type: ErrorTypeEnum;
+  reason: string;
+}
+export type { ErrorType };

packages/flow-client/models/ErrorTypeEnum.ts

@@ -0,0 +1,14 @@
+type ErrorTypeEnum =
+  | 'asr_error'
+  | 'protocol_error'
+  | 'config_error'
+  | 'idle_timeout'
+  | 'session_timeout'
+  | 'not_allowed'
+  | 'not_authorised'
+  | 'quota_exceeded'
+  | 'timelimit_exceeded'
+  | 'job_error'
+  | 'internal_error'
+  | 'unknown_error';
+export type { ErrorTypeEnum };

packages/flow-client/models/FlowClientMessage.ts

@@ -0,0 +1,12 @@
+import type { StartConversation } from './StartConversation';
+import type { AudioReceived } from './AudioReceived';
+import type { AudioEnded } from './AudioEnded';
+import type { AddInput } from './AddInput';
+import type { ToolResult } from './ToolResult';
+type FlowClientMessage =
+  | StartConversation
+  | AudioReceived
+  | AudioEnded
+  | AddInput
+  | ToolResult;
+export type { FlowClientMessage };

packages/flow-client/models/FlowServerMessage.ts

@@ -0,0 +1,36 @@
+import type { ConversationStarted } from './ConversationStarted';
+import type { AudioAdded } from './AudioAdded';
+import type { AddPartialTranscript } from './AddPartialTranscript';
+import type { AddTranscript } from './AddTranscript';
+import type { ResponseStarted } from './ResponseStarted';
+import type { ResponseCompleted } from './ResponseCompleted';
+import type { ResponseInterrupted } from './ResponseInterrupted';
+import type { ToolInvoke } from './ToolInvoke';
+import type { ErrorType } from './ErrorType';
+import type { Warning } from './Warning';
+import type { ConversationTerminationWarning } from './ConversationTerminationWarning';
+import type { Info } from './Info';
+import type { StatusUpdateInfo } from './StatusUpdateInfo';
+import type { ConversationDurationLimitInfo } from './ConversationDurationLimitInfo';
+import type { ConcurrentSessionUsageInfo } from './ConcurrentSessionUsageInfo';
+import type { ConversationEnding } from './ConversationEnding';
+import type { ConversationEnded } from './ConversationEnded';
+type FlowServerMessage =
+  | ConversationStarted
+  | AudioAdded
+  | AddPartialTranscript
+  | AddTranscript
+  | ResponseStarted
+  | ResponseCompleted
+  | ResponseInterrupted
+  | ToolInvoke
+  | ErrorType
+  | Warning
+  | ConversationTerminationWarning
+  | Info
+  | StatusUpdateInfo
+  | ConversationDurationLimitInfo
+  | ConcurrentSessionUsageInfo
+  | ConversationEnding
+  | ConversationEnded;
+export type { FlowServerMessage };

packages/flow-client/models/Info.ts

@@ -0,0 +1,6 @@
+interface Info {
+  reason: string;
+  message: 'Info';
+  type: string;
+}
+export type { Info };

packages/flow-client/models/LanguagePackInfo.ts

@@ -0,0 +1,9 @@
+import type { WritingDirectionEnum } from './WritingDirectionEnum';
+interface LanguagePackInfo {
+  language_description?: string;
+  word_delimiter: string;
+  writing_direction?: WritingDirectionEnum;
+  itn?: boolean;
+  adapted?: boolean;
+}
+export type { LanguagePackInfo };

packages/flow-client/models/RecognitionAlternative.ts

@@ -0,0 +1,9 @@
+import type { RecognitionDisplay } from './RecognitionDisplay';
+interface RecognitionAlternative {
+  content: string;
+  confidence: number;
+  language?: string;
+  display?: RecognitionDisplay;
+  speaker?: string;
+}
+export type { RecognitionAlternative };

packages/flow-client/models/RecognitionDisplay.ts

@@ -0,0 +1,5 @@
+import type { RecognitionDisplayDirectionEnum } from './RecognitionDisplayDirectionEnum';
+interface RecognitionDisplay {
+  direction: RecognitionDisplayDirectionEnum;
+}
+export type { RecognitionDisplay };

packages/flow-client/models/RecognitionDisplayDirectionEnum.ts

@@ -0,0 +1,2 @@
+type RecognitionDisplayDirectionEnum = 'ltr' | 'rtl';
+export type { RecognitionDisplayDirectionEnum };

packages/flow-client/models/RecognitionMetadata.ts

@@ -0,0 +1,6 @@
+interface RecognitionMetadata {
+  start_time: number;
+  end_time: number;
+  transcript: string;
+}
+export type { RecognitionMetadata };

packages/flow-client/models/RecognitionResult.ts

@@ -0,0 +1,15 @@
+import type { TypeEnum } from './TypeEnum';
+import type { AttachesToEnum } from './AttachesToEnum';
+import type { RecognitionAlternative } from './RecognitionAlternative';
+interface RecognitionResult {
+  type: TypeEnum;
+  start_time: number;
+  end_time: number;
+  channel?: string;
+  attaches_to?: AttachesToEnum;
+  is_eos?: boolean;
+  alternatives?: RecognitionAlternative[];
+  score?: number;
+  volume?: number;
+}
+export type { RecognitionResult };

packages/flow-client/models/ResponseCompleted.ts

@@ -0,0 +1,7 @@
+interface ResponseCompleted {
+  message: 'ResponseCompleted';
+  content: string;
+  start_time: number;
+  end_time: number;
+}
+export type { ResponseCompleted };

packages/flow-client/models/ResponseInterrupted.ts

@@ -0,0 +1,7 @@
+interface ResponseInterrupted {
+  message: 'ResponseInterrupted';
+  content: string;
+  start_time: number;
+  end_time: number;
+}
+export type { ResponseInterrupted };

packages/flow-client/models/ResponseStarted.ts

@@ -0,0 +1,6 @@
+interface ResponseStarted {
+  message: 'ResponseStarted';
+  content: string;
+  start_time: number;
+}
+export type { ResponseStarted };

packages/flow-client/models/StartConversation.ts

@@ -0,0 +1,12 @@
+import type { AudioFormat } from './AudioFormat';
+import type { ConversationConfig } from './ConversationConfig';
+import type { ToolConfig } from './ToolConfig';
+import type { DebugObject } from './DebugObject';
+interface StartConversation {
+  message: 'StartConversation';
+  audio_format?: AudioFormat;
+  conversation_config: ConversationConfig;
+  tools?: ToolConfig[];
+  debug?: DebugObject;
+}
+export type { StartConversation };

packages/flow-client/models/StatusEnum.ts

@@ -0,0 +1,2 @@
+type StatusEnum = 'ok' | 'rejected' | 'failed';
+export type { StatusEnum };

packages/flow-client/models/StatusUpdateInfo.ts

@@ -0,0 +1,8 @@
+import type { StatusUpdateInfoEvent } from './StatusUpdateInfoEvent';
+interface StatusUpdateInfo {
+  reason: string;
+  message: 'Info';
+  event: StatusUpdateInfoEvent;
+  type: 'status_update';
+}
+export type { StatusUpdateInfo };

packages/flow-client/models/StatusUpdateInfoEvent.ts

@@ -0,0 +1,5 @@
+interface StatusUpdateInfoEvent {
+  prev_status: string;
+  status: string;
+}
+export type { StatusUpdateInfoEvent };

packages/flow-client/models/ToolConfig.ts

@@ -0,0 +1,7 @@
+import type { ToolTypeEnum } from './ToolTypeEnum';
+import type { ToolFunctionConfig } from './ToolFunctionConfig';
+interface ToolConfig {
+  type: ToolTypeEnum;
+  function: ToolFunctionConfig;
+}
+export type { ToolConfig };

packages/flow-client/models/ToolFunction.ts

@@ -0,0 +1,5 @@
+interface ToolFunction {
+  name: string;
+  arguments: { [name: string]: string };
+}
+export type { ToolFunction };