feat: update file processor (#1254)

js/Testing.MD

@@ -10,6 +10,7 @@ The test suite is designed to run across multiple environments. Before running t
    - GitHub integration is enabled and active
    - Gmail integration is enabled and active 
    - CodeInterpreter is configured for the default entity
+   - Github drive and file_id is configured for the default entity
 
 3. Trigger Configuration
    - At least one trigger must be active in the system

js/config/getTestConfig.ts

@@ -4,6 +4,9 @@ const CURRENT_FILE_DIR = __dirname;
 export type BACKEND_CONFIG = {
     COMPOSIO_API_KEY: string;
     BACKEND_HERMES_URL: string;
+    drive: {
+        downloadable_file_id: string;
+    }
 }
 
 export const getTestConfig = (): BACKEND_CONFIG => {

js/config/test.config.local.json

@@ -1,4 +1,7 @@
 {
     "COMPOSIO_API_KEY": "",
-    "BACKEND_HERMES_URL": "http://localhost:9900"
+    "BACKEND_HERMES_URL": "http://localhost:9900",
+    "drive":{
+        "downloadable_file_id":"18rcI9N7cJRG15E2qyWXtNSFeDg4Rj-T3"
+    }
 }

js/config/test.config.prod.json

@@ -1,4 +1,7 @@
 {
     "COMPOSIO_API_KEY": "pv7s0lpq7z5vu27cikyls",
-    "BACKEND_HERMES_URL": "https://backend.composio.dev"
+    "BACKEND_HERMES_URL": "https://backend.composio.dev",
+    "drive":{
+        "downloadable_file_id":"18rcI9N7cJRG15E2qyWXtNSFeDg4Rj-T3"
+    }
 }

js/config/test.config.staging.json

@@ -1,4 +1,7 @@
 {
     "COMPOSIO_API_KEY": "gxpf3a5v864651jp741heq",
-    "BACKEND_HERMES_URL": "https://staging-backend.composio.dev"
+    "BACKEND_HERMES_URL": "https://staging-backend.composio.dev",
+    "drive":{
+        "downloadable_file_id":"18rcI9N7cJRG15E2qyWXtNSFeDg4Rj-T3"
+    }
 }

js/openapi-ts.config.js

@@ -3,7 +3,7 @@ import { defineConfig } from "@hey-api/openapi-ts";
 
 export default defineConfig({
     client: "@hey-api/client-axios",
-    input: "http://localhost:9900/openapi.json",
+    input: "https://backend.composio.dev/openapi.json",
     output: "src/sdk/client",
     services: {
         asClass: true,

js/src/sdk/base.toolset.spec.ts

@@ -160,16 +160,17 @@ describe("ComposioToolSet class tests", () => {
     const ACTION_NAME = "GMAIL_SEND_EMAIL";
     const actions = await toolset.getToolsSchema({ actions: [ACTION_NAME] });
 
+    const firstAction = actions[0]!;
     // Check if exist
     expect(
-      actions[0]!.parameters.properties["attachment_file_uri_path"]
+      firstAction.parameters.properties["attachment_schema_parsed_file"]
     ).toBeDefined();
 
     const requestBody = {
       recipient_email: "[email protected]",
       subject: "Test email from himanshu",
       body: "This is a test email",
-      attachment_file_uri_path:
+      attachment_schema_parsed_file:
         "https://composio.dev/wp-content/uploads/2024/07/Composio-Logo.webp",
     };
 
@@ -184,6 +185,20 @@ describe("ComposioToolSet class tests", () => {
     expect(executionResult.data).toBeDefined();
   });
 
+  it("should execute downloadable file action", async () => {
+    const ACTION_NAME = "GOOGLEDRIVE_PARSE_FILE";
+    const executionResult = await toolset.executeAction({
+      action: ACTION_NAME,
+      params: {
+        file_id: testConfig.drive.downloadable_file_id,
+      },
+      entityId: "default",
+    });
+
+    // @ts-ignore
+    expect(executionResult.data.file.uri.length).toBeGreaterThan(0);
+  });
+
   it("should get tools with usecase limit", async () => {
     const tools = await toolset.getToolsSchema({
       useCase: "follow user",

js/src/sdk/base.toolset.ts

@@ -20,17 +20,17 @@ import { ActionExecutionResDto } from "./client/types.gen";
 import { ActionExecuteResponse, Actions } from "./models/actions";
 import { ActiveTriggers } from "./models/activeTriggers";
 import { Apps } from "./models/apps";
-import { BackendClient } from "./models/backendClient";
+import { AxiosBackendClient } from "./models/backendClient";
 import { ConnectedAccounts } from "./models/connectedAccounts";
 import { Integrations } from "./models/integrations";
 import { Triggers } from "./models/triggers";
 import { getUserDataJson } from "./utils/config";
 import { CEG } from "./utils/error";
 import { COMPOSIO_SDK_ERROR_CODES } from "./utils/errors/src/constants";
 import {
-  fileInputProcessor,
-  fileResponseProcessor,
-  fileSchemaProcessor,
+  FILE_DOWNLOADABLE_PROCESSOR,
+  FILE_INPUT_PROCESSOR,
+  FILE_SCHEMA_PROCESSOR,
 } from "./utils/processor/file";
 
 export type ExecuteActionParams = z.infer<typeof ZExecuteActionParams> & {
@@ -45,7 +45,7 @@ export class ComposioToolSet {
   entityId: string = "default";
   connectedAccountIds: Record<string, string> = {};
 
-  backendClient: BackendClient;
+  backendClient: AxiosBackendClient;
   connectedAccounts: ConnectedAccounts;
   apps: Apps;
   actions: Actions;
@@ -60,9 +60,9 @@ export class ComposioToolSet {
     post: TPostProcessor[];
     schema: TSchemaProcessor[];
   } = {
-    pre: [fileInputProcessor],
-    post: [fileResponseProcessor],
-    schema: [fileSchemaProcessor],
+    pre: [FILE_INPUT_PROCESSOR],
+    post: [FILE_DOWNLOADABLE_PROCESSOR],
+    schema: [FILE_SCHEMA_PROCESSOR],
   };
 
   private userDefinedProcessors: {
@@ -169,7 +169,7 @@ export class ComposioToolSet {
       }
     }
 
-    const apps = await this.client.actions.list({
+    const appActions = await this.client.actions.list({
       apps: parsedFilters.apps?.join(","),
       tags: parsedFilters.tags?.join(","),
       useCase: parsedFilters.useCase,
@@ -191,25 +191,31 @@ export class ComposioToolSet {
       );
     });
 
-    const toolsActions = [...(apps?.items || []), ...toolsWithCustomActions];
+    const toolsActions = [
+      ...(appActions?.items || []),
+      ...toolsWithCustomActions,
+    ];
 
     const allSchemaProcessor = [
       ...this.internalProcessors.schema,
       ...(this.userDefinedProcessors.schema
         ? [this.userDefinedProcessors.schema]
         : []),
     ];
-
-    return toolsActions.map((tool) => {
+    const processedTools = [];
+    // Iterate over the tools and process them
+    for (const tool of toolsActions) {
       let schema = tool as RawActionData;
-      allSchemaProcessor.forEach((processor) => {
-        schema = processor({
+      // Process the schema with all the processors
+      for (const processor of allSchemaProcessor) {
+        schema = await processor({
           actionName: schema?.name,
           toolSchema: schema,
         });
-      });
-      return schema;
-    });
+      }
+      processedTools.push(schema);
+    }
+    return processedTools;
   }
 
   async createAction<P extends Parameters = z.ZodObject<{}>>(
@@ -265,7 +271,7 @@ export class ComposioToolSet {
     ];
 
     for (const processor of allInputProcessor) {
-      params = processor({
+      params = await processor({
         params: params,
         actionName: action,
       });
@@ -324,9 +330,10 @@ export class ComposioToolSet {
         : []),
     ];
 
-    let dataToReturn = { ...data };
+    // Dirty way to avoid copy
+    let dataToReturn = JSON.parse(JSON.stringify(data));
     for (const processor of allOutputProcessor) {
-      dataToReturn = processor({
+      dataToReturn = await processor({
         actionName: meta.action,
         toolResponse: dataToReturn,
       });