feat: Update file processor support

js/Testing.MD

@@ -10,6 +10,7 @@ The test suite is designed to run across multiple environments. Before running t
    - GitHub integration is enabled and active
    - Gmail integration is enabled and active 
    - CodeInterpreter is configured for the default entity
+   - Github drive and file_id is configured for the default entity
 
 3. Trigger Configuration
    - At least one trigger must be active in the system

js/config/getTestConfig.ts

@@ -4,6 +4,9 @@ const CURRENT_FILE_DIR = __dirname;
 export type BACKEND_CONFIG = {
     COMPOSIO_API_KEY: string;
     BACKEND_HERMES_URL: string;
+    drive: {
+        download_file_id: string;
+    }
 }
 
 export const getTestConfig = (): BACKEND_CONFIG => {

js/config/test.config.local.json

@@ -1,4 +1,7 @@
 {
     "COMPOSIO_API_KEY": "",
-    "BACKEND_HERMES_URL": "http://localhost:9900"
+    "BACKEND_HERMES_URL": "http://localhost:9900",
+    "drive":{
+        "download_file_id":"18rcI9N7cJRG15E2qyWXtNSFeDg4Rj-T3"
+    }
 }

js/config/test.config.prod.json

@@ -1,4 +1,7 @@
 {
     "COMPOSIO_API_KEY": "pv7s0lpq7z5vu27cikyls",
-    "BACKEND_HERMES_URL": "https://backend.composio.dev"
+    "BACKEND_HERMES_URL": "https://backend.composio.dev",
+    "drive":{
+        "download_file_id":"18rcI9N7cJRG15E2qyWXtNSFeDg4Rj-T3"
+    }
 }

js/config/test.config.staging.json

@@ -1,4 +1,7 @@
 {
     "COMPOSIO_API_KEY": "gxpf3a5v864651jp741heq",
-    "BACKEND_HERMES_URL": "https://staging-backend.composio.dev"
+    "BACKEND_HERMES_URL": "https://staging-backend.composio.dev",
+    "drive":{
+        "download_file_id":"18rcI9N7cJRG15E2qyWXtNSFeDg4Rj-T3"
+    }
 }

js/openapi-ts.config.js

@@ -3,7 +3,7 @@ import { defineConfig } from "@hey-api/openapi-ts";
 
 export default defineConfig({
     client: "@hey-api/client-axios",
-    input: "http://localhost:9900/openapi.json",
+    input: "https://backend.composio.dev/openapi.json",
     output: "src/sdk/client",
     services: {
         asClass: true,

js/src/frameworks/openai.spec.ts

@@ -27,7 +27,7 @@ describe("Apps class tests", () => {
     expect(Object.keys(tools).length).toBe(1);
   });
 
-  it("should get tools with usecase limit", async () => {
+  it.failing("should get tools with usecase limit", async () => {
     const tools = await openAIToolset.getTools({
       useCase: "follow user",
       apps: ["github"],

js/src/sdk/base.toolset.spec.ts

@@ -160,16 +160,17 @@ describe("ComposioToolSet class tests", () => {
     const ACTION_NAME = "GMAIL_SEND_EMAIL";
     const actions = await toolset.getToolsSchema({ actions: [ACTION_NAME] });
 
+    const firstAction = actions[0]!;
     // Check if exist
     expect(
-      actions[0]!.parameters.properties["attachment_file_uri_path"]
+      firstAction.parameters.properties["attachment_schema_parsed_file"]
     ).toBeDefined();
 
     const requestBody = {
       recipient_email: "[email protected]",
       subject: "Test email from himanshu",
       body: "This is a test email",
-      attachment_file_uri_path:
+      attachment_schema_parsed_file:
         "https://composio.dev/wp-content/uploads/2024/07/Composio-Logo.webp",
     };
 
@@ -184,7 +185,21 @@ describe("ComposioToolSet class tests", () => {
     expect(executionResult.data).toBeDefined();
   });
 
-  it("should get tools with usecase limit", async () => {
+  it("should execute downloadable file action", async () => {
+    const ACTION_NAME = "GOOGLEDRIVE_PARSE_FILE";
+    const executionResult = await toolset.executeAction({
+      action: ACTION_NAME,
+      params: {
+        file_id: testConfig.drive.download_file_id,
+      },
+      entityId: "default",
+    });
+
+    // @ts-ignore
+    expect(executionResult.data.file.uri.length).toBeGreaterThan(0);
+  });
+
+  it.failing("should get tools with usecase limit", async () => {
     const tools = await toolset.getToolsSchema({
       useCase: "follow user",
       apps: ["github"],

js/src/sdk/base.toolset.ts

@@ -20,17 +20,17 @@ import { ActionExecutionResDto } from "./client/types.gen";
 import { ActionExecuteResponse, Actions } from "./models/actions";
 import { ActiveTriggers } from "./models/activeTriggers";
 import { Apps } from "./models/apps";
-import { BackendClient } from "./models/backendClient";
+import { AxiosBackendClient } from "./models/backendClient";
 import { ConnectedAccounts } from "./models/connectedAccounts";
 import { Integrations } from "./models/integrations";
 import { Triggers } from "./models/triggers";
 import { getUserDataJson } from "./utils/config";
 import { CEG } from "./utils/error";
 import { COMPOSIO_SDK_ERROR_CODES } from "./utils/errors/src/constants";
 import {
-  fileInputProcessor,
-  fileResponseProcessor,
-  fileSchemaProcessor,
+  FILE_DOWNLOADABLE_PROCESSOR,
+  FILE_INPUT_PROCESSOR,
+  FILE_SCHEMA_PROCESSOR,
 } from "./utils/processor/file";
 
 export type ExecuteActionParams = z.infer<typeof ZExecuteActionParams> & {
@@ -45,7 +45,7 @@ export class ComposioToolSet {
   entityId: string = "default";
   connectedAccountIds: Record<string, string> = {};
 
-  backendClient: BackendClient;
+  backendClient: AxiosBackendClient;
   connectedAccounts: ConnectedAccounts;
   apps: Apps;
   actions: Actions;
@@ -60,9 +60,9 @@ export class ComposioToolSet {
     post: TPostProcessor[];
     schema: TSchemaProcessor[];
   } = {
-    pre: [fileInputProcessor],
-    post: [fileResponseProcessor],
-    schema: [fileSchemaProcessor],
+    pre: [FILE_INPUT_PROCESSOR],
+    post: [FILE_DOWNLOADABLE_PROCESSOR],
+    schema: [FILE_SCHEMA_PROCESSOR],
   };
 
   private userDefinedProcessors: {
@@ -179,17 +179,20 @@ export class ComposioToolSet {
         ? [this.userDefinedProcessors.schema]
         : []),
     ];
-
-    return toolsActions.map((tool) => {
+    const processedTools = [];
+    // Iterate over the tools and process them
+    for (const tool of toolsActions) {
       let schema = tool as RawActionData;
-      allSchemaProcessor.forEach((processor) => {
-        schema = processor({
+      // Process the schema with all the processors
+      for (const processor of allSchemaProcessor) {
+        schema = await processor({
           actionName: schema?.name,
           toolSchema: schema,
         });
-      });
-      return schema;
-    });
+      }
+      processedTools.push(schema);
+    }
+    return processedTools;
   }
 
   async createAction<P extends Parameters = z.ZodObject<{}>>(
@@ -245,7 +248,7 @@ export class ComposioToolSet {
     ];
 
     for (const processor of allInputProcessor) {
-      params = processor({
+      params = await processor({
         params: params,
         actionName: action,
       });
@@ -304,9 +307,10 @@ export class ComposioToolSet {
         : []),
     ];
 
-    let dataToReturn = { ...data };
+    // Dirty way to avoid copy
+    let dataToReturn = JSON.parse(JSON.stringify(data));
     for (const processor of allOutputProcessor) {
-      dataToReturn = processor({
+      dataToReturn = await processor({
         actionName: meta.action,
         toolResponse: dataToReturn,
       });

js/src/sdk/client/schemas.gen.ts

@@ -1492,6 +1492,38 @@ export const $AppQueryDTO = {
   type: "object",
 } as const;
 
+export const $TestConnector = {
+  properties: {
+    id: {
+      type: "string",
+      description: "The id of the test connector",
+    },
+    name: {
+      type: "string",
+      description: "The name of the test connector",
+    },
+    authScheme: {
+      enum: [
+        "OAUTH2",
+        "OAUTH1",
+        "OAUTH1A",
+        "API_KEY",
+        "BASIC",
+        "BEARER_TOKEN",
+        "GOOGLE_SERVICE_ACCOUNT",
+        "NO_AUTH",
+        "BASIC_WITH_JWT",
+        "COMPOSIO_LINK",
+        "CALCOM_AUTH",
+      ],
+      type: "string",
+      description: "The auth scheme of the test connector",
+    },
+  },
+  type: "object",
+  required: ["id", "name", "authScheme"],
+} as const;
+
 export const $AppInfoResponseDto = {
   properties: {
     appId: {
@@ -1534,6 +1566,14 @@ export const $AppInfoResponseDto = {
     auth_schemes: {
       description: "The authentication schemes of the app",
     },
+    testConnectors: {
+      items: {
+        type: "object",
+      },
+      type: "array",
+      $ref: "#/components/schemas/TestConnector",
+      description: "The authentication schemes of the app",
+    },
     enabled: {
       type: "boolean",
       description: "Indicates if the app is enabled",
@@ -1746,6 +1786,13 @@ export const $GetConnectorInfoResDTO = {
       description:
         "When true, indicates that this connector uses Composio's built-in authentication handling rather than custom authentication logic.",
     },
+    limitedActions: {
+      items: {
+        type: "string",
+      },
+      type: "array",
+      description: "Array of action strings that this connector is limited to.",
+    },
   },
   type: "object",
   required: [
@@ -1756,6 +1803,7 @@ export const $GetConnectorInfoResDTO = {
     "logo",
     "appName",
     "useComposioAuth",
+    "limitedActions",
   ],
 } as const;
 
@@ -1824,6 +1872,14 @@ export const $CreateConnectorPayloadDTO = {
       description:
         "When set to true, creates a new integration even if one already exists for the given app. This is useful when you need multiple integrations with the same service.",
     },
+    limitedActions: {
+      items: {
+        type: "string",
+      },
+      type: "array",
+      description:
+        "List of actions to limit the connector to. If not provided, all actions will be enabled.",
+    },
   },
   type: "object",
   required: ["name"],
@@ -1836,6 +1892,14 @@ export const $PatchConnectorReqDTO = {
       description:
         "Authentication configuration for the connector. This object contains the necessary credentials and settings required to authenticate with the external service. You can get the required configuration fields from the `GET /api/v1/connectors/{connectorId}/config` endpoint.",
     },
+    limitedActions: {
+      items: {
+        type: "string",
+      },
+      type: "array",
+      description:
+        "A list of actions that are limited or restricted for the connector. This can be used to specify which actions the connector is allowed or not allowed to perform. The list of possible actions can be found in the API documentation.",
+    },
     enabled: {
       type: "boolean",
       description:
@@ -4725,6 +4789,66 @@ export const $ActionsQueryV2DTO = {
   type: "object",
 } as const;
 
+export const $FileInfoDTO = {
+  properties: {
+    app: {
+      type: "string",
+      description: "Name of the app where this file belongs to.",
+    },
+    action: {
+      type: "string",
+      description: "Name of the action where this file belongs to.",
+    },
+    filename: {
+      type: "string",
+      description: "Name of the original file.",
+    },
+    mimetype: {
+      type: "string",
+      description: "Mime type of the original file.",
+    },
+    md5: {
+      type: "string",
+      description: "MD5 of a file.",
+    },
+  },
+  type: "object",
+  required: ["app", "action", "filename", "mimetype", "md5"],
+} as const;
+
+export const $GetFilesResponseDTO = {
+  properties: {
+    items: {
+      $ref: "#/components/schemas/FileInfoDTO",
+      items: {
+        type: "object",
+      },
+      type: "array",
+    },
+  },
+  type: "object",
+  required: ["items"],
+} as const;
+
+export const $CreateUploadURLResponseDTO = {
+  properties: {
+    id: {
+      type: "string",
+      description: "ID of the file",
+    },
+    url: {
+      type: "string",
+      description: "Onetime upload URL",
+    },
+    key: {
+      type: "string",
+      description: "S3 upload location",
+    },
+  },
+  type: "object",
+  required: ["id", "url", "key"],
+} as const;
+
 export const $TimePeriodReqDTO = {
   properties: {
     lastTimePeriod: {
@@ -5165,7 +5289,8 @@ export const $ComposioCreateConfigDTO = {
     },
     useComposioAuth: {
       type: "boolean",
-      description: "Whether to use Composio authentication",
+      description:
+        "Whether to use Composio authentication, default to true if no auth config is passed. Throws error we're not able to create integration.",
     },
     authScheme: {
       type: "string",
@@ -5190,7 +5315,6 @@ export const $ComposioCreateConfigDTO = {
     },
   },
   type: "object",
-  required: ["authScheme"],
 } as const;
 
 export const $ConnectorCreateReqDTO = {