From 64edf89d33fc4352e2e0e9f623868d10298a5d70 Mon Sep 17 00:00:00 2001 From: Himanshu Dixit Date: Tue, 11 Feb 2025 23:18:35 +0530 Subject: [PATCH] feat: update file processor (#1254) --- js/Testing.MD | 1 + js/config/getTestConfig.ts | 3 + js/config/test.config.local.json | 5 +- js/config/test.config.prod.json | 5 +- js/config/test.config.staging.json | 5 +- js/openapi-ts.config.js | 2 +- js/src/sdk/base.toolset.spec.ts | 19 ++- js/src/sdk/base.toolset.ts | 47 +++--- js/src/sdk/client/schemas.gen.ts | 128 ++++++++++++++- js/src/sdk/client/services.gen.ts | 38 +++++ js/src/sdk/client/types.gen.ts | 142 ++++++++++++++--- js/src/sdk/index.ts | 6 +- js/src/sdk/models/Entity.ts | 6 +- js/src/sdk/models/actions.ts | 6 +- js/src/sdk/models/activeTriggers.ts | 6 +- js/src/sdk/models/apps.ts | 6 +- js/src/sdk/models/backendClient.spec.ts | 13 +- js/src/sdk/models/backendClient.ts | 2 +- js/src/sdk/models/connectedAccounts.ts | 8 +- js/src/sdk/models/integrations.ts | 6 +- js/src/sdk/models/triggers.ts | 6 +- js/src/sdk/testUtils/getBackendClient.ts | 6 +- js/src/sdk/utils/errors/src/composioError.ts | 27 ++-- js/src/sdk/utils/errors/src/formatter.ts | 2 +- js/src/sdk/utils/processor/file.ts | 158 +++++++++++-------- js/src/sdk/utils/processor/fileUtils.ts | 111 +++++++++++++ js/src/types/base_toolset.ts | 6 +- js/src/utils/logger.ts | 2 +- 28 files changed, 604 insertions(+), 168 deletions(-) create mode 100644 js/src/sdk/utils/processor/fileUtils.ts diff --git a/js/Testing.MD b/js/Testing.MD index 10d29fa73d5..03eee6ca0dc 100644 --- a/js/Testing.MD +++ b/js/Testing.MD @@ -10,6 +10,7 @@ The test suite is designed to run across multiple environments. Before running t - GitHub integration is enabled and active - Gmail integration is enabled and active - CodeInterpreter is configured for the default entity + - Github drive and file_id is configured for the default entity 3. Trigger Configuration - At least one trigger must be active in the system diff --git a/js/config/getTestConfig.ts b/js/config/getTestConfig.ts index f36d1fe9672..abbefd83b41 100644 --- a/js/config/getTestConfig.ts +++ b/js/config/getTestConfig.ts @@ -4,6 +4,9 @@ const CURRENT_FILE_DIR = __dirname; export type BACKEND_CONFIG = { COMPOSIO_API_KEY: string; BACKEND_HERMES_URL: string; + drive: { + downloadable_file_id: string; + } } export const getTestConfig = (): BACKEND_CONFIG => { diff --git a/js/config/test.config.local.json b/js/config/test.config.local.json index 04910d0fb52..7498e96b3e0 100644 --- a/js/config/test.config.local.json +++ b/js/config/test.config.local.json @@ -1,4 +1,7 @@ { "COMPOSIO_API_KEY": "", - "BACKEND_HERMES_URL": "http://localhost:9900" + "BACKEND_HERMES_URL": "http://localhost:9900", + "drive":{ + "downloadable_file_id":"18rcI9N7cJRG15E2qyWXtNSFeDg4Rj-T3" + } } diff --git a/js/config/test.config.prod.json b/js/config/test.config.prod.json index a7be0317a8a..0cffc3b3b7e 100644 --- a/js/config/test.config.prod.json +++ b/js/config/test.config.prod.json @@ -1,4 +1,7 @@ { "COMPOSIO_API_KEY": "pv7s0lpq7z5vu27cikyls", - "BACKEND_HERMES_URL": "https://backend.composio.dev" + "BACKEND_HERMES_URL": "https://backend.composio.dev", + "drive":{ + "downloadable_file_id":"18rcI9N7cJRG15E2qyWXtNSFeDg4Rj-T3" + } } \ No newline at end of file diff --git a/js/config/test.config.staging.json b/js/config/test.config.staging.json index fd68c232bce..ded79a818b6 100644 --- a/js/config/test.config.staging.json +++ b/js/config/test.config.staging.json @@ -1,4 +1,7 @@ { "COMPOSIO_API_KEY": "gxpf3a5v864651jp741heq", - "BACKEND_HERMES_URL": "https://staging-backend.composio.dev" + "BACKEND_HERMES_URL": "https://staging-backend.composio.dev", + "drive":{ + "downloadable_file_id":"18rcI9N7cJRG15E2qyWXtNSFeDg4Rj-T3" + } } diff --git a/js/openapi-ts.config.js b/js/openapi-ts.config.js index 77c9ed9d5d2..0e4700f042e 100644 --- a/js/openapi-ts.config.js +++ b/js/openapi-ts.config.js @@ -3,7 +3,7 @@ import { defineConfig } from "@hey-api/openapi-ts"; export default defineConfig({ client: "@hey-api/client-axios", - input: "http://localhost:9900/openapi.json", + input: "https://backend.composio.dev/openapi.json", output: "src/sdk/client", services: { asClass: true, diff --git a/js/src/sdk/base.toolset.spec.ts b/js/src/sdk/base.toolset.spec.ts index ad7520ef2db..9674672c610 100644 --- a/js/src/sdk/base.toolset.spec.ts +++ b/js/src/sdk/base.toolset.spec.ts @@ -160,16 +160,17 @@ describe("ComposioToolSet class tests", () => { const ACTION_NAME = "GMAIL_SEND_EMAIL"; const actions = await toolset.getToolsSchema({ actions: [ACTION_NAME] }); + const firstAction = actions[0]!; // Check if exist expect( - actions[0]!.parameters.properties["attachment_file_uri_path"] + firstAction.parameters.properties["attachment_schema_parsed_file"] ).toBeDefined(); const requestBody = { recipient_email: "himanshu@composio.dev", subject: "Test email from himanshu", body: "This is a test email", - attachment_file_uri_path: + attachment_schema_parsed_file: "https://composio.dev/wp-content/uploads/2024/07/Composio-Logo.webp", }; @@ -184,6 +185,20 @@ describe("ComposioToolSet class tests", () => { expect(executionResult.data).toBeDefined(); }); + it("should execute downloadable file action", async () => { + const ACTION_NAME = "GOOGLEDRIVE_PARSE_FILE"; + const executionResult = await toolset.executeAction({ + action: ACTION_NAME, + params: { + file_id: testConfig.drive.downloadable_file_id, + }, + entityId: "default", + }); + + // @ts-ignore + expect(executionResult.data.file.uri.length).toBeGreaterThan(0); + }); + it("should get tools with usecase limit", async () => { const tools = await toolset.getToolsSchema({ useCase: "follow user", diff --git a/js/src/sdk/base.toolset.ts b/js/src/sdk/base.toolset.ts index 72acf427469..71a6150393a 100644 --- a/js/src/sdk/base.toolset.ts +++ b/js/src/sdk/base.toolset.ts @@ -20,7 +20,7 @@ import { ActionExecutionResDto } from "./client/types.gen"; import { ActionExecuteResponse, Actions } from "./models/actions"; import { ActiveTriggers } from "./models/activeTriggers"; import { Apps } from "./models/apps"; -import { BackendClient } from "./models/backendClient"; +import { AxiosBackendClient } from "./models/backendClient"; import { ConnectedAccounts } from "./models/connectedAccounts"; import { Integrations } from "./models/integrations"; import { Triggers } from "./models/triggers"; @@ -28,9 +28,9 @@ import { getUserDataJson } from "./utils/config"; import { CEG } from "./utils/error"; import { COMPOSIO_SDK_ERROR_CODES } from "./utils/errors/src/constants"; import { - fileInputProcessor, - fileResponseProcessor, - fileSchemaProcessor, + FILE_DOWNLOADABLE_PROCESSOR, + FILE_INPUT_PROCESSOR, + FILE_SCHEMA_PROCESSOR, } from "./utils/processor/file"; export type ExecuteActionParams = z.infer & { @@ -45,7 +45,7 @@ export class ComposioToolSet { entityId: string = "default"; connectedAccountIds: Record = {}; - backendClient: BackendClient; + backendClient: AxiosBackendClient; connectedAccounts: ConnectedAccounts; apps: Apps; actions: Actions; @@ -60,9 +60,9 @@ export class ComposioToolSet { post: TPostProcessor[]; schema: TSchemaProcessor[]; } = { - pre: [fileInputProcessor], - post: [fileResponseProcessor], - schema: [fileSchemaProcessor], + pre: [FILE_INPUT_PROCESSOR], + post: [FILE_DOWNLOADABLE_PROCESSOR], + schema: [FILE_SCHEMA_PROCESSOR], }; private userDefinedProcessors: { @@ -169,7 +169,7 @@ export class ComposioToolSet { } } - const apps = await this.client.actions.list({ + const appActions = await this.client.actions.list({ apps: parsedFilters.apps?.join(","), tags: parsedFilters.tags?.join(","), useCase: parsedFilters.useCase, @@ -191,7 +191,10 @@ export class ComposioToolSet { ); }); - const toolsActions = [...(apps?.items || []), ...toolsWithCustomActions]; + const toolsActions = [ + ...(appActions?.items || []), + ...toolsWithCustomActions, + ]; const allSchemaProcessor = [ ...this.internalProcessors.schema, @@ -199,17 +202,20 @@ export class ComposioToolSet { ? [this.userDefinedProcessors.schema] : []), ]; - - return toolsActions.map((tool) => { + const processedTools = []; + // Iterate over the tools and process them + for (const tool of toolsActions) { let schema = tool as RawActionData; - allSchemaProcessor.forEach((processor) => { - schema = processor({ + // Process the schema with all the processors + for (const processor of allSchemaProcessor) { + schema = await processor({ actionName: schema?.name, toolSchema: schema, }); - }); - return schema; - }); + } + processedTools.push(schema); + } + return processedTools; } async createAction

>( @@ -265,7 +271,7 @@ export class ComposioToolSet { ]; for (const processor of allInputProcessor) { - params = processor({ + params = await processor({ params: params, actionName: action, }); @@ -324,9 +330,10 @@ export class ComposioToolSet { : []), ]; - let dataToReturn = { ...data }; + // Dirty way to avoid copy + let dataToReturn = JSON.parse(JSON.stringify(data)); for (const processor of allOutputProcessor) { - dataToReturn = processor({ + dataToReturn = await processor({ actionName: meta.action, toolResponse: dataToReturn, }); diff --git a/js/src/sdk/client/schemas.gen.ts b/js/src/sdk/client/schemas.gen.ts index d59ef218ecb..d65c91adc21 100644 --- a/js/src/sdk/client/schemas.gen.ts +++ b/js/src/sdk/client/schemas.gen.ts @@ -1492,6 +1492,38 @@ export const $AppQueryDTO = { type: "object", } as const; +export const $TestConnector = { + properties: { + id: { + type: "string", + description: "The id of the test connector", + }, + name: { + type: "string", + description: "The name of the test connector", + }, + authScheme: { + enum: [ + "OAUTH2", + "OAUTH1", + "OAUTH1A", + "API_KEY", + "BASIC", + "BEARER_TOKEN", + "GOOGLE_SERVICE_ACCOUNT", + "NO_AUTH", + "BASIC_WITH_JWT", + "COMPOSIO_LINK", + "CALCOM_AUTH", + ], + type: "string", + description: "The auth scheme of the test connector", + }, + }, + type: "object", + required: ["id", "name", "authScheme"], +} as const; + export const $AppInfoResponseDto = { properties: { appId: { @@ -1534,6 +1566,14 @@ export const $AppInfoResponseDto = { auth_schemes: { description: "The authentication schemes of the app", }, + testConnectors: { + items: { + type: "object", + }, + type: "array", + $ref: "#/components/schemas/TestConnector", + description: "The authentication schemes of the app", + }, enabled: { type: "boolean", description: "Indicates if the app is enabled", @@ -1746,6 +1786,13 @@ export const $GetConnectorInfoResDTO = { description: "When true, indicates that this connector uses Composio's built-in authentication handling rather than custom authentication logic.", }, + limitedActions: { + items: { + type: "string", + }, + type: "array", + description: "Array of action strings that this connector is limited to.", + }, }, type: "object", required: [ @@ -1756,6 +1803,7 @@ export const $GetConnectorInfoResDTO = { "logo", "appName", "useComposioAuth", + "limitedActions", ], } as const; @@ -1824,6 +1872,14 @@ export const $CreateConnectorPayloadDTO = { description: "When set to true, creates a new integration even if one already exists for the given app. This is useful when you need multiple integrations with the same service.", }, + limitedActions: { + items: { + type: "string", + }, + type: "array", + description: + "List of actions to limit the connector to. If not provided, all actions will be enabled.", + }, }, type: "object", required: ["name"], @@ -1836,6 +1892,14 @@ export const $PatchConnectorReqDTO = { description: "Authentication configuration for the connector. This object contains the necessary credentials and settings required to authenticate with the external service. You can get the required configuration fields from the `GET /api/v1/connectors/{connectorId}/config` endpoint.", }, + limitedActions: { + items: { + type: "string", + }, + type: "array", + description: + "A list of actions that are limited or restricted for the connector. This can be used to specify which actions the connector is allowed or not allowed to perform. The list of possible actions can be found in the API documentation.", + }, enabled: { type: "boolean", description: @@ -4725,6 +4789,66 @@ export const $ActionsQueryV2DTO = { type: "object", } as const; +export const $FileInfoDTO = { + properties: { + app: { + type: "string", + description: "Name of the app where this file belongs to.", + }, + action: { + type: "string", + description: "Name of the action where this file belongs to.", + }, + filename: { + type: "string", + description: "Name of the original file.", + }, + mimetype: { + type: "string", + description: "Mime type of the original file.", + }, + md5: { + type: "string", + description: "MD5 of a file.", + }, + }, + type: "object", + required: ["app", "action", "filename", "mimetype", "md5"], +} as const; + +export const $GetFilesResponseDTO = { + properties: { + items: { + $ref: "#/components/schemas/FileInfoDTO", + items: { + type: "object", + }, + type: "array", + }, + }, + type: "object", + required: ["items"], +} as const; + +export const $CreateUploadURLResponseDTO = { + properties: { + id: { + type: "string", + description: "ID of the file", + }, + url: { + type: "string", + description: "Onetime upload URL", + }, + key: { + type: "string", + description: "S3 upload location", + }, + }, + type: "object", + required: ["id", "url", "key"], +} as const; + export const $TimePeriodReqDTO = { properties: { lastTimePeriod: { @@ -5165,7 +5289,8 @@ export const $ComposioCreateConfigDTO = { }, useComposioAuth: { type: "boolean", - description: "Whether to use Composio authentication", + description: + "Whether to use Composio authentication, default to true if no auth config is passed. Throws error we're not able to create integration.", }, authScheme: { type: "string", @@ -5190,7 +5315,6 @@ export const $ComposioCreateConfigDTO = { }, }, type: "object", - required: ["authScheme"], } as const; export const $ConnectorCreateReqDTO = { diff --git a/js/src/sdk/client/services.gen.ts b/js/src/sdk/client/services.gen.ts index ffff0e27844..945e1cdcc4f 100644 --- a/js/src/sdk/client/services.gen.ts +++ b/js/src/sdk/client/services.gen.ts @@ -6,6 +6,8 @@ import { type Options, } from "@hey-api/client-axios"; import type { + ActionsControllerV2ListUserFilesError, + ActionsControllerV2ListUserFilesResponse, AddProjectData, AddProjectError, AddProjectResponse, @@ -18,6 +20,9 @@ import type { CreateConnectorV2Data, CreateConnectorV2Error, CreateConnectorV2Response, + CreateFileUploadUrlData, + CreateFileUploadUrlError, + CreateFileUploadUrlResponse, CreateProjectData, CreateProjectError, CreateProjectResponse, @@ -700,6 +705,39 @@ export class ActionsService { url: "/api/v2/actions/search/advanced", }); } + + /** + * List user files + */ + public static v2ListUserFiles( + options?: Options + ) { + return (options?.client ?? client).get< + ActionsControllerV2ListUserFilesResponse, + ActionsControllerV2ListUserFilesError, + ThrowOnError + >({ + ...options, + url: "/api/v2/actions/files/list", + }); + } + + /** + * Create file upload url + * Create file upload URL for action execution. + */ + public static createFileUploadUrl( + options: Options + ) { + return (options?.client ?? client).post< + CreateFileUploadUrlResponse, + CreateFileUploadUrlError, + ThrowOnError + >({ + ...options, + url: "/api/v2/actions/files/upload/{fileType}", + }); + } } export class ConnectionsService { diff --git a/js/src/sdk/client/types.gen.ts b/js/src/sdk/client/types.gen.ts index d97865d6d0d..4e6fa40aef4 100644 --- a/js/src/sdk/client/types.gen.ts +++ b/js/src/sdk/client/types.gen.ts @@ -1068,6 +1068,48 @@ export type includeLocal = "true" | "false"; */ export type sortBy = "alphabet" | "usage" | "no_sort"; +export type TestConnector = { + /** + * The id of the test connector + */ + id: string; + /** + * The name of the test connector + */ + name: string; + /** + * The auth scheme of the test connector + */ + authScheme: + | "OAUTH2" + | "OAUTH1" + | "OAUTH1A" + | "API_KEY" + | "BASIC" + | "BEARER_TOKEN" + | "GOOGLE_SERVICE_ACCOUNT" + | "NO_AUTH" + | "BASIC_WITH_JWT" + | "COMPOSIO_LINK" + | "CALCOM_AUTH"; +}; + +/** + * The auth scheme of the test connector + */ +export type authScheme = + | "OAUTH2" + | "OAUTH1" + | "OAUTH1A" + | "API_KEY" + | "BASIC" + | "BEARER_TOKEN" + | "GOOGLE_SERVICE_ACCOUNT" + | "NO_AUTH" + | "BASIC_WITH_JWT" + | "COMPOSIO_LINK" + | "CALCOM_AUTH"; + export type AppInfoResponseDto = { /** * Unique identifier (UUID) for the app @@ -1101,6 +1143,10 @@ export type AppInfoResponseDto = { * The authentication schemes of the app */ auth_schemes?: unknown; + /** + * The authentication schemes of the app + */ + testConnectors?: TestConnector; /** * Indicates if the app is enabled */ @@ -1234,9 +1280,9 @@ export type GetConnectorInfoResDTO = { */ useComposioAuth: boolean; /** - * List of actions that are limited to this connector, trying to execute any other action apart from these will throw an unauthorized error + * Array of action strings that this connector is limited to. */ - limitedActions: string[]; + limitedActions: Array; }; /** @@ -1282,6 +1328,10 @@ export type CreateConnectorPayloadDTO = { * When set to true, creates a new integration even if one already exists for the given app. This is useful when you need multiple integrations with the same service. */ forceNewIntegration?: boolean; + /** + * List of actions to limit the connector to. If not provided, all actions will be enabled. + */ + limitedActions?: Array; }; export type PatchConnectorReqDTO = { @@ -1291,6 +1341,10 @@ export type PatchConnectorReqDTO = { authConfig?: { [key: string]: unknown; }; + /** + * A list of actions that are limited or restricted for the connector. This can be used to specify which actions the connector is allowed or not allowed to perform. The list of possible actions can be found in the API documentation. + */ + limitedActions?: Array; /** * Flag to indicate if the connector is enabled. When set to false, the connector will not process any requests. You can toggle this value to temporarily disable the connector without deleting it. Default value can be found in the `GET /api/v1/connectors/{connectorId}` endpoint response. */ @@ -1663,10 +1717,6 @@ export type Parameter = { value: string; }; -/** - * The location of the parameter. Can be 'query' or 'header'. - */ - export type Data = { /** * First field of the data object. @@ -3251,6 +3301,48 @@ export type ActionsQueryV2DTO = { sortBy?: "alphabet" | "usage" | "no_sort"; }; +export type FileInfoDTO = { + /** + * Name of the app where this file belongs to. + */ + app: string; + /** + * Name of the action where this file belongs to. + */ + action: string; + /** + * Name of the original file. + */ + filename: string; + /** + * Mime type of the original file. + */ + mimetype: string; + /** + * MD5 of a file. + */ + md5: string; +}; + +export type GetFilesResponseDTO = { + items: FileInfoDTO; +}; + +export type CreateUploadURLResponseDTO = { + /** + * ID of the file + */ + id: string; + /** + * Onetime upload URL + */ + url: string; + /** + * S3 upload location + */ + key: string; +}; + export type TimePeriodReqDTO = { /** * Time period to get the data for @@ -3529,22 +3621,6 @@ export type ComposioSearchConfigDTO = { | "CALCOM_AUTH"; }; -/** - * Authentication scheme to use - */ -export type authScheme = - | "OAUTH2" - | "OAUTH1" - | "OAUTH1A" - | "API_KEY" - | "BASIC" - | "BEARER_TOKEN" - | "GOOGLE_SERVICE_ACCOUNT" - | "NO_AUTH" - | "BASIC_WITH_JWT" - | "COMPOSIO_LINK" - | "CALCOM_AUTH"; - export type ConnectorSearchFilterDTOV2 = { /** * Filter options for the connector @@ -3582,13 +3658,13 @@ export type ComposioCreateConfigDTO = { */ name?: string; /** - * Whether to use Composio authentication + * Whether to use Composio authentication, default to true if no auth config is passed. Throws error we're not able to create integration. */ useComposioAuth?: boolean; /** * Authentication scheme to use */ - authScheme: + authScheme?: | "OAUTH2" | "OAUTH1" | "OAUTH1A" @@ -4016,6 +4092,24 @@ export type AdvancedUseCaseSearchResponse2 = AdvancedUseCaseSearchResponse; export type AdvancedUseCaseSearchError = unknown; +export type ActionsControllerV2ListUserFilesResponse = GetFilesResponseDTO; + +export type ActionsControllerV2ListUserFilesError = unknown; + +export type CreateFileUploadUrlData = { + /** + * FileInfoDTO + */ + body?: FileInfoDTO; + path: { + fileType: unknown; + }; +}; + +export type CreateFileUploadUrlResponse = CreateUploadURLResponseDTO; + +export type CreateFileUploadUrlError = unknown; + export type ListConnectionsData = { query?: { appNames?: string; diff --git a/js/src/sdk/index.ts b/js/src/sdk/index.ts index f70bfa5b13e..d127193274f 100644 --- a/js/src/sdk/index.ts +++ b/js/src/sdk/index.ts @@ -11,7 +11,7 @@ import { Entity } from "./models/Entity"; import { Actions } from "./models/actions"; import { ActiveTriggers } from "./models/activeTriggers"; import { Apps } from "./models/apps"; -import { BackendClient } from "./models/backendClient"; +import { AxiosBackendClient } from "./models/backendClient"; import { ConnectedAccounts } from "./models/connectedAccounts"; import { Integrations } from "./models/integrations"; import { Triggers } from "./models/triggers"; @@ -36,7 +36,7 @@ export class Composio { * It provides access to various models that allow for operations on connected accounts, apps, * actions, triggers, integrations, and active triggers. */ - backendClient: BackendClient; + backendClient: AxiosBackendClient; connectedAccounts: ConnectedAccounts; apps: Apps; actions: Actions; @@ -94,7 +94,7 @@ export class Composio { ); // Initialize the BackendClient with the parsed API key and base URL. - this.backendClient = new BackendClient( + this.backendClient = new AxiosBackendClient( apiKeyParsed, baseURLParsed, config?.runtime diff --git a/js/src/sdk/models/Entity.ts b/js/src/sdk/models/Entity.ts index b83acda1ec3..5f0e429e4a8 100644 --- a/js/src/sdk/models/Entity.ts +++ b/js/src/sdk/models/Entity.ts @@ -13,7 +13,7 @@ import { TELEMETRY_EVENTS } from "../utils/telemetry/events"; import { ActionExecuteResponse, Actions } from "./actions"; import { ActiveTriggers } from "./activeTriggers"; import { Apps } from "./apps"; -import { BackendClient } from "./backendClient"; +import { AxiosBackendClient } from "./backendClient"; import { ConnectedAccounts, ConnectionItem, @@ -43,7 +43,7 @@ export type ConnectedAccountListRes = GetConnectionsResponseDto; export class Entity { id: string; - private backendClient: BackendClient; + private backendClient: AxiosBackendClient; private triggerModel: Triggers; private actionsModel: Actions; private apps: Apps; @@ -53,7 +53,7 @@ export class Entity { private fileName: string = "js/src/sdk/models/Entity.ts"; - constructor(backendClient: BackendClient, id: string = "default") { + constructor(backendClient: AxiosBackendClient, id: string = "default") { this.backendClient = backendClient; this.id = id; this.triggerModel = new Triggers(this.backendClient); diff --git a/js/src/sdk/models/actions.ts b/js/src/sdk/models/actions.ts index 231594dee67..0744e32a511 100644 --- a/js/src/sdk/models/actions.ts +++ b/js/src/sdk/models/actions.ts @@ -18,7 +18,7 @@ import { import { CEG } from "../utils/error"; import { TELEMETRY_LOGGER } from "../utils/telemetry"; import { TELEMETRY_EVENTS } from "../utils/telemetry/events"; -import { BackendClient } from "./backendClient"; +import { AxiosBackendClient } from "./backendClient"; /** * Request types inferred from zod schemas @@ -43,10 +43,10 @@ export type ActionFindActionEnumsByUseCaseRes = Array; export class Actions { // Remove this as we might not need it - private backendClient: BackendClient; + private backendClient: AxiosBackendClient; fileName: string = "js/src/sdk/models/actions.ts"; - constructor(backendClient: BackendClient) { + constructor(backendClient: AxiosBackendClient) { this.backendClient = backendClient; } diff --git a/js/src/sdk/models/activeTriggers.ts b/js/src/sdk/models/activeTriggers.ts index 59b16564ec2..4c334a2da03 100644 --- a/js/src/sdk/models/activeTriggers.ts +++ b/js/src/sdk/models/activeTriggers.ts @@ -8,7 +8,7 @@ import { import { CEG } from "../utils/error"; import { TELEMETRY_LOGGER } from "../utils/telemetry"; import { TELEMETRY_EVENTS } from "../utils/telemetry/events"; -import { BackendClient } from "./backendClient"; +import { AxiosBackendClient } from "./backendClient"; export type TriggerItemParam = z.infer; export type GetActiveTriggersData = z.infer; @@ -16,9 +16,9 @@ export type TriggerItemRes = z.infer; export type TriggerChangeResponse = { status: string }; export class ActiveTriggers { // Remove this as we might not need it - private backendClient: BackendClient; + private backendClient: AxiosBackendClient; private fileName: string = "js/src/sdk/models/activeTriggers.ts"; - constructor(backendClient: BackendClient) { + constructor(backendClient: AxiosBackendClient) { this.backendClient = backendClient; } diff --git a/js/src/sdk/models/apps.ts b/js/src/sdk/models/apps.ts index 1c046b406bf..75955d1b9d8 100644 --- a/js/src/sdk/models/apps.ts +++ b/js/src/sdk/models/apps.ts @@ -16,7 +16,7 @@ import { ZRequiredParamsFullResponse, ZRequiredParamsResponse, } from "../types/app"; -import { BackendClient } from "./backendClient"; +import { AxiosBackendClient } from "./backendClient"; // schema types generated from zod export type AppGetRequiredParams = z.infer; @@ -42,9 +42,9 @@ export type AppListRes = AppListResDTO; export type AppItemListResponse = AppInfoResponseDto; export class Apps { - private backendClient: BackendClient; + private backendClient: AxiosBackendClient; private fileName: string = "js/src/sdk/models/apps.ts"; - constructor(backendClient: BackendClient) { + constructor(backendClient: AxiosBackendClient) { this.backendClient = backendClient; } diff --git a/js/src/sdk/models/backendClient.spec.ts b/js/src/sdk/models/backendClient.spec.ts index 5188073619c..643622fb875 100644 --- a/js/src/sdk/models/backendClient.spec.ts +++ b/js/src/sdk/models/backendClient.spec.ts @@ -1,6 +1,6 @@ import { beforeAll, describe, expect, it } from "@jest/globals"; import { BACKEND_CONFIG, getTestConfig } from "../../../config/getTestConfig"; -import { BackendClient } from "./backendClient"; +import { AxiosBackendClient } from "./backendClient"; describe("Apps class tests", () => { let _backendClient; @@ -11,21 +11,22 @@ describe("Apps class tests", () => { }); it("should create an Apps instance and retrieve apps list", async () => { - _backendClient = new BackendClient( + _backendClient = new AxiosBackendClient( testConfig.COMPOSIO_API_KEY, testConfig.BACKEND_HERMES_URL ); }); it("should throw an error if api key is not provided", async () => { - expect(() => new BackendClient("", testConfig.BACKEND_HERMES_URL)).toThrow( - "API key is not available" - ); + expect( + () => new AxiosBackendClient("", testConfig.BACKEND_HERMES_URL) + ).toThrow("API key is not available"); }); it("should throw and error if wrong base url is provided", async () => { expect( - () => new BackendClient(testConfig.COMPOSIO_API_KEY, "htt://wrong.url") + () => + new AxiosBackendClient(testConfig.COMPOSIO_API_KEY, "htt://wrong.url") ).toThrow("🔗 Base URL htt://wrong.url is not valid"); }); }); diff --git a/js/src/sdk/models/backendClient.ts b/js/src/sdk/models/backendClient.ts index e19a9f00401..27adf4cbc28 100644 --- a/js/src/sdk/models/backendClient.ts +++ b/js/src/sdk/models/backendClient.ts @@ -9,7 +9,7 @@ import { removeTrailingSlashIfExists } from "../utils/string"; /** * Class representing the details required to initialize and configure the API client. */ -export class BackendClient { +export class AxiosBackendClient { /** * The API key used for authenticating requests. */ diff --git a/js/src/sdk/models/connectedAccounts.ts b/js/src/sdk/models/connectedAccounts.ts index 4dbad32b225..569fc709124 100644 --- a/js/src/sdk/models/connectedAccounts.ts +++ b/js/src/sdk/models/connectedAccounts.ts @@ -17,7 +17,7 @@ import { ZAuthMode } from "../types/integration"; import { CEG } from "../utils/error"; import { TELEMETRY_LOGGER } from "../utils/telemetry"; import { TELEMETRY_EVENTS } from "../utils/telemetry/events"; -import { BackendClient } from "./backendClient"; +import { AxiosBackendClient } from "./backendClient"; type ConnectedAccountsListData = z.infer & { /** @deprecated use appUniqueKeys field instead */ @@ -48,14 +48,14 @@ export type ConnectionItem = ConnectionParams; * Class representing connected accounts in the system. */ export class ConnectedAccounts { - private backendClient: BackendClient; + private backendClient: AxiosBackendClient; private fileName: string = "js/src/sdk/models/connectedAccounts.ts"; /** * Initializes a new instance of the ConnectedAccounts class. - * @param {BackendClient} backendClient - The backend client instance. + * @param {AxiosBackendClient} backendClient - The backend client instance. */ - constructor(backendClient: BackendClient) { + constructor(backendClient: AxiosBackendClient) { this.backendClient = backendClient; } diff --git a/js/src/sdk/models/integrations.ts b/js/src/sdk/models/integrations.ts index 9bfff996a23..8a4daf5aa2a 100644 --- a/js/src/sdk/models/integrations.ts +++ b/js/src/sdk/models/integrations.ts @@ -18,7 +18,7 @@ import { COMPOSIO_SDK_ERROR_CODES } from "../utils/errors/src/constants"; import { TELEMETRY_LOGGER } from "../utils/telemetry"; import { TELEMETRY_EVENTS } from "../utils/telemetry/events"; import { Apps } from "./apps"; -import { BackendClient } from "./backendClient"; +import { AxiosBackendClient } from "./backendClient"; // Types generated from zod schemas @@ -48,11 +48,11 @@ export type IntegrationRequiredParamsRes = ExpectedInputFieldsDTO[]; export type IntegrationDeleteRes = DeleteRowAPIDTO; export class Integrations { - private backendClient: BackendClient; + private backendClient: AxiosBackendClient; private fileName: string = "js/src/sdk/models/integrations.ts"; private apps: Apps; - constructor(backendClient: BackendClient) { + constructor(backendClient: AxiosBackendClient) { this.backendClient = backendClient; this.apps = new Apps(backendClient); } diff --git a/js/src/sdk/models/triggers.ts b/js/src/sdk/models/triggers.ts index 50d48e5c6c2..01cf4645934 100644 --- a/js/src/sdk/models/triggers.ts +++ b/js/src/sdk/models/triggers.ts @@ -1,6 +1,6 @@ import logger from "../../utils/logger"; import { PusherUtils, TriggerData } from "../utils/pusher"; -import { BackendClient } from "./backendClient"; +import { AxiosBackendClient } from "./backendClient"; import apiClient from "../client/client"; @@ -58,9 +58,9 @@ export type SingleInstanceTriggerParam = z.infer< export class Triggers { trigger_to_client_event = "trigger_to_client"; - private backendClient: BackendClient; + private backendClient: AxiosBackendClient; private fileName: string = "js/src/sdk/models/triggers.ts"; - constructor(backendClient: BackendClient) { + constructor(backendClient: AxiosBackendClient) { this.backendClient = backendClient; } diff --git a/js/src/sdk/testUtils/getBackendClient.ts b/js/src/sdk/testUtils/getBackendClient.ts index 3427c7e9078..84a87323fb3 100644 --- a/js/src/sdk/testUtils/getBackendClient.ts +++ b/js/src/sdk/testUtils/getBackendClient.ts @@ -1,7 +1,7 @@ import { getTestConfig } from "../../../config/getTestConfig"; -import { BackendClient } from "../models/backendClient"; +import { AxiosBackendClient } from "../models/backendClient"; -export const getBackendClient = (): BackendClient => { +export const getBackendClient = (): AxiosBackendClient => { const testConfig = getTestConfig(); if (testConfig["COMPOSIO_API_KEY"] === undefined) { throw new Error("COMPOSIO_API_KEY is not set in the test config"); @@ -11,5 +11,5 @@ export const getBackendClient = (): BackendClient => { } const COMPOSIO_API_KEY = testConfig["COMPOSIO_API_KEY"]; const BACKEND_HERMES_URL = testConfig["BACKEND_HERMES_URL"]; - return new BackendClient(COMPOSIO_API_KEY, BACKEND_HERMES_URL); + return new AxiosBackendClient(COMPOSIO_API_KEY, BACKEND_HERMES_URL); }; diff --git a/js/src/sdk/utils/errors/src/composioError.ts b/js/src/sdk/utils/errors/src/composioError.ts index b1b7c96112e..0e15e8cfb1f 100644 --- a/js/src/sdk/utils/errors/src/composioError.ts +++ b/js/src/sdk/utils/errors/src/composioError.ts @@ -1,6 +1,6 @@ import { logError } from ".."; import { getUUID } from "../../../../utils/common"; -import { getLogLevel } from "../../../../utils/logger"; +import logger, { getLogLevel, LOG_LEVELS } from "../../../../utils/logger"; /** * Custom error class for Composio that provides rich error details, tracking, and improved debugging @@ -67,19 +67,22 @@ export class ComposioError extends Error { } } - // eslint-disable-next-line no-console - console.log( - `🚀 [Info] Give Feedback / Get Help: https://dub.composio.dev/discord ` - ); - // eslint-disable-next-line no-console - console.log( - `🐛 [Info] Create a new issue: https://github.com/ComposioHQ/composio/issues ` - ); - if (getLogLevel() !== "debug") { + // Only in case of info or debug, we will log the error + if (LOG_LEVELS[getLogLevel()] >= 2) { // eslint-disable-next-line no-console - console.log( - `⛔ [Info] If you need to debug this error, set env variable COMPOSIO_LOGGING_LEVEL=debug` + logger.info( + `🚀 [Info] Give Feedback / Get Help: https://dub.composio.dev/discord ` ); + // eslint-disable-next-line no-console + logger.info( + `🐛 [Info] Create a new issue: https://github.com/ComposioHQ/composio/issues ` + ); + if (getLogLevel() !== "debug") { + // eslint-disable-next-line no-console + logger.info( + `⛔ [Info] If you need to debug this error, set env variable COMPOSIO_LOGGING_LEVEL=debug` + ); + } } logError({ diff --git a/js/src/sdk/utils/errors/src/formatter.ts b/js/src/sdk/utils/errors/src/formatter.ts index 4bf5a9951da..da12e72aaaf 100644 --- a/js/src/sdk/utils/errors/src/formatter.ts +++ b/js/src/sdk/utils/errors/src/formatter.ts @@ -104,7 +104,7 @@ export const generateMetadataFromAxiosError = ( metadata?: Record; } ): Record => { - const requestId = axiosError.response?.headers["x-request-id"]; + const requestId = axiosError.request?.headers["x-request-id"]; return { fullUrl: (axiosError.config?.baseURL ?? "") + (axiosError.config?.url ?? ""), diff --git a/js/src/sdk/utils/processor/file.ts b/js/src/sdk/utils/processor/file.ts index ad82d4c1dde..7bc97764def 100644 --- a/js/src/sdk/utils/processor/file.ts +++ b/js/src/sdk/utils/processor/file.ts @@ -3,89 +3,119 @@ import { TPreProcessor, TSchemaProcessor, } from "../../../types/base_toolset"; -import logger from "../../../utils/logger"; -import { saveFile } from "../fileUtils"; +import { downloadFileFromS3, getFileDataAfterUploadingToS3 } from "./fileUtils"; -export const fileResponseProcessor: TPostProcessor = ({ +type FileBasePropertySchema = { + type: string; + title: string; + description: string; + file_uploadable?: boolean; +} & Record; + +const FILE_SUFFIX = "_schema_parsed_file"; + +const convertFileSchemaProperty = ( + key: string, + property: FileBasePropertySchema +) => { + if (!property.file_uploadable) { + return property; + } + + return { + keyName: `${key}${FILE_SUFFIX}`, + type: "string", + description: property.description, + }; +}; + +const processFileUpload = async ( + params: Record, + actionName: string +) => { + const result = { ...params }; + + for (const [key, value] of Object.entries(result)) { + if (!key.endsWith(FILE_SUFFIX)) continue; + + const originalKey = key.replace(FILE_SUFFIX, ""); + const fileData = await getFileDataAfterUploadingToS3( + value as string, + actionName + ); + + result[originalKey] = fileData; + delete result[key]; + } + + return result; +}; + +export const FILE_INPUT_PROCESSOR: TPreProcessor = async ({ + params, + actionName, +}) => { + return processFileUpload(params, actionName); +}; + +export const FILE_DOWNLOADABLE_PROCESSOR: TPostProcessor = async ({ actionName, toolResponse, }) => { - const responseData = - (toolResponse.data.response_data as Record) || {}; - const fileData = responseData.file as - | { name: string; content: string } - | undefined; + const result = JSON.parse(JSON.stringify(toolResponse)); - if (!fileData) return toolResponse; + for (const [key, value] of Object.entries(toolResponse.data)) { + const fileData = value as { s3url?: string; mimetype?: string }; - const fileNamePrefix = `${actionName}_${Date.now()}`; - const filePath = saveFile(fileNamePrefix, fileData.content, true); + if (!fileData?.s3url) continue; - delete responseData.file; + const downloadedFile = await downloadFileFromS3({ + actionName, + s3Url: fileData.s3url, + mimeType: fileData.mimetype || "application/txt", + }); - return { - ...toolResponse, - data: { - ...toolResponse.data, - file_uri_path: filePath, - }, - }; + result.data[key] = { + uri: downloadedFile.filePath, + mimeType: downloadedFile.mimeType, + }; + } + + return result; }; -export const fileInputProcessor: TPreProcessor = ({ params, actionName }) => { - const requestData = Object.entries(params).reduce( - (acc, [key, value]) => { - if (key === "file_uri_path" && typeof value === "string") { - try { - //eslint-disable-next-line @typescript-eslint/no-require-imports - const fileContent = require("fs").readFileSync(value, "utf-8"); - const fileName = - value.split("/").pop() || `${actionName}_${Date.now()}`; - acc["file"] = { name: fileName, content: fileContent }; - } catch (error) { - logger.error(`Error reading file at ${value}:`, error); - acc["file"] = { name: value, content: "" }; // Fallback to original value if reading fails - } - } else { - acc[key] = value; - } - return acc; - }, - {} as Record - ); +export const FILE_SCHEMA_PROCESSOR: TSchemaProcessor = ({ toolSchema }) => { + const { properties, required: requiredProps = [] } = toolSchema.parameters; + const newProperties = { ...properties }; + const newRequired = [...requiredProps]; - return requestData; -}; + for (const [key, property] of Object.entries(newProperties)) { + if (!property.file_uploadable) continue; -export const fileSchemaProcessor: TSchemaProcessor = ({ toolSchema }) => { - const { properties } = toolSchema.parameters; - const clonedProperties = JSON.parse(JSON.stringify(properties)); - - for (const propertyKey of Object.keys(clonedProperties)) { - const object = clonedProperties[propertyKey]; - const isObject = typeof object === "object"; - const isFile = - isObject && - object?.required?.includes("name") && - object?.required?.includes("content"); - - if (isFile) { - const newKey = `${propertyKey}_file_uri_path`; - clonedProperties[newKey] = { - type: "string", - title: "Name", - description: "Local absolute path to the file or http url to the file", - }; - - delete clonedProperties[propertyKey]; + const { type, keyName, description } = convertFileSchemaProperty( + key, + property as FileBasePropertySchema + ); + + newProperties[keyName as string] = { + title: property.title, + type, + description, + }; + + if (requiredProps.includes(key)) { + newRequired[newRequired.indexOf(key)] = keyName as string; } + + delete newProperties[key]; } return { ...toolSchema, parameters: { ...toolSchema.parameters, - properties: clonedProperties, + properties: newProperties, + required: newRequired, }, }; }; diff --git a/js/src/sdk/utils/processor/fileUtils.ts b/js/src/sdk/utils/processor/fileUtils.ts new file mode 100644 index 00000000000..8ac43693b8d --- /dev/null +++ b/js/src/sdk/utils/processor/fileUtils.ts @@ -0,0 +1,111 @@ +import axios, { AxiosError } from "axios"; +import crypto from "crypto"; +import apiClient from "../../client/client"; +import { saveFile } from "../fileUtils"; + +const readFileContent = async ( + path: string +): Promise<{ content: string; mimeType: string }> => { + try { + const content = require("fs").readFileSync(path, "utf-8"); + return { content, mimeType: "text/plain" }; + } catch (error) { + throw new Error(`Error reading file at ${path}: ${error}`); + } +}; + +const readFileContentFromURL = async ( + path: string +): Promise<{ content: string; mimeType: string }> => { + const response = await fetch(path); + const content = await response.text(); + const mimeType = response.headers.get("content-type") || "text/plain"; + return { content, mimeType }; +}; + +const uploadFileToS3 = async ( + content: string, + actionName: string, + appName: string, + mimeType: string +): Promise => { + const response = await apiClient.actionsV2.createFileUploadUrl({ + body: { + action: actionName, + app: appName, + filename: `${actionName}_${Date.now()}`, + mimetype: mimeType, + md5: crypto.createHash("md5").update(content).digest("hex"), + }, + path: { + fileType: "request", + }, + }); + + const data = response.data as unknown as { url: string; key: string }; + const signedURL = data!.url; + const s3key = data!.key; + + try { + // Upload the file to the S3 bucket + await axios.put(signedURL, content); + } catch (e) { + const error = e as AxiosError; + // if error is 403, then continue + if (error instanceof AxiosError && error.response?.status === 403) { + return signedURL; + } + throw new Error(`Error uploading file to S3: ${error}`); + } + + return s3key; +}; + +export const getFileDataAfterUploadingToS3 = async ( + path: string, + actionName: string +): Promise<{ + name: string; + mimetype: string; + s3key: string; +}> => { + const isURL = path.startsWith("http"); + const fileData = isURL + ? await readFileContentFromURL(path) + : await readFileContent(path); + + const s3key = await uploadFileToS3( + fileData.content, + actionName, + actionName, + fileData.mimeType + ); + + return { + name: path.split("/").pop() || `${actionName}_${Date.now()}`, + mimetype: fileData.mimeType, + s3key: s3key, + }; +}; + +export const downloadFileFromS3 = async ({ + actionName, + s3Url, + mimeType, +}: { + actionName: string; + s3Url: string; + mimeType: string; +}) => { + const response = await axios.get(s3Url); + + const extension = mimeType.split("/")[1] || "txt"; + const fileName = `${actionName}_${Date.now()}`; + const filePath = saveFile(fileName, response.data, true); + return { + name: fileName, + mimeType: mimeType, + s3Key: s3Url, + filePath: filePath, + }; +}; diff --git a/js/src/types/base_toolset.ts b/js/src/types/base_toolset.ts index 8b61568e297..a791655cc43 100644 --- a/js/src/types/base_toolset.ts +++ b/js/src/types/base_toolset.ts @@ -52,7 +52,7 @@ export type TPreProcessor = ({ }: { params: Record; actionName: string; -}) => Record; +}) => Promise> | Record; export type TPostProcessor = ({ actionName, @@ -60,7 +60,7 @@ export type TPostProcessor = ({ }: { actionName: string; toolResponse: ActionExecutionResDto; -}) => ActionExecutionResDto; +}) => Promise | ActionExecutionResDto; export type TSchemaProcessor = ({ actionName, @@ -68,7 +68,7 @@ export type TSchemaProcessor = ({ }: { actionName: string; toolSchema: RawActionData; -}) => RawActionData; +}) => Promise | RawActionData; export const ZToolSchemaFilter = z.object({ actions: z.array(z.string()).optional(), diff --git a/js/src/utils/logger.ts b/js/src/utils/logger.ts index ab1a1f6e020..53250f0976d 100644 --- a/js/src/utils/logger.ts +++ b/js/src/utils/logger.ts @@ -1,7 +1,7 @@ import { getEnvVariable } from "./shared"; // Define log levels with corresponding priorities -const LOG_LEVELS = { +export const LOG_LEVELS = { error: 0, // Highest priority - critical errors warn: 1, // Warning messages info: 2, // General information