From 848ddd9471c3f62e9ec5f1056a053559a56a94a2 Mon Sep 17 00:00:00 2001 From: yashdamani Date: Tue, 15 Jul 2025 09:29:36 +0530 Subject: [PATCH] [MaximJS] together ai sdk integration --- package.json | 12 + .../logger/together/togetherLogger.test.ts | 399 ++++++++++++++++++ src/lib/logger/together/utils.ts | 80 ++++ src/lib/logger/together/wrapper.ts | 288 +++++++++++++ together-ai-sdk.ts | 2 + 5 files changed, 781 insertions(+) create mode 100644 src/lib/logger/together/togetherLogger.test.ts create mode 100644 src/lib/logger/together/utils.ts create mode 100644 src/lib/logger/together/wrapper.ts create mode 100644 together-ai-sdk.ts diff --git a/package.json b/package.json index 0c81186..7fc5e97 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,18 @@ "./vercel-ai-sdk": { "types": "./vercel-ai-sdk.d.ts", "default": "./vercel-ai-sdk.js" + }, + "./together-ai-sdk": { + "types": "./together-ai-sdk.d.ts", + "default": "./together-ai-sdk.js" + }, + "./groq-sdk": { + "types": "./groq-sdk.d.ts", + "default": "./groq-sdk.js" } + }, + "dependencies": { + "groq-sdk": "^0.26.0", + "together-ai": "^0.19.0" } } diff --git a/src/lib/logger/together/togetherLogger.test.ts b/src/lib/logger/together/togetherLogger.test.ts new file mode 100644 index 0000000..ece5e0c --- /dev/null +++ b/src/lib/logger/together/togetherLogger.test.ts @@ -0,0 +1,399 @@ +import { Together } from "together-ai"; +import { config } from "dotenv"; +import { v4 as uuid } from "uuid"; +import { z } from "zod"; +import { Maxim } from "../../../../index"; +import { wrapMaximTogetherClient } from "../../../../together-ai-sdk"; + +config(); + +let maxim: Maxim; + +const togetherApiKey = process.env['TOGETHER_API_KEY'] || ""; +const apiKey = process.env['MAXIM_API_KEY'] || ""; +const baseUrl = process.env['MAXIM_BASE_URL'] || ""; +const repoId = process.env['MAXIM_LOG_REPO_ID'] || ""; + +describe("Comprehensive MaximTogetherTracer Tests", () => { + beforeAll(async () => { + if (!baseUrl || !apiKey || !repoId || !togetherApiKey) { + throw new Error("MAXIM_BASE_URL, MAXIM_API_KEY, TOGETHER_API_KEY & LOG_REPO_ID environment variables are required"); + } + maxim = new Maxim({ + baseUrl: baseUrl, + apiKey: apiKey, + }); + }); + + afterAll(async () => { + await maxim.cleanup(); + }); + + describe("Basic Chat Model Tests", () => { + // it("should trace Together chat model with basic text and system message", async () => { + // if (!repoId || !togetherApiKey) { + // throw new Error("MAXIM_LOG_REPO_ID and TOGETHER_API_KEY environment variables are required"); + // } + // const logger = await maxim.logger({ id: repoId }); + // if (!logger) { + // throw new Error("Logger is not available"); + // } + // const client = wrapMaximTogetherClient(new Together({ apiKey: togetherApiKey }), logger); + + // const query = "Who is Sachin Tendulkar?"; + // try { + // const response = await client.completions.create({ + // model: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", + // temperature: 0.3, + // top_p: 1, + // frequency_penalty: 0, + // prompt: query, + // max_tokens: 4096, + // }); + // console.log("Together response for basic generateText", JSON.stringify(response.choices[0].text)); + // } catch (error) { + // console.error(error); + // } + // }, 40000); + + // it("should log the user message and Together chat model response for multiple messages", async () => { + // if (!repoId || !togetherApiKey) { + // throw new Error("MAXIM_LOG_REPO_ID and TOGETHER_API_KEY environment variables are required"); + // } + // const logger = await maxim.logger({ id: repoId }); + // if (!logger) { + // throw new Error("Logger is not available"); + // } + // const client = wrapMaximTogetherClient(new Together({ apiKey: togetherApiKey }), logger); + + // try { + // const result = await client.chat.completions.create({ + // model: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", + // temperature: 0.3, + // top_p: 1, + // frequency_penalty: 0, + // messages: [ + // { + // role: "system", + // content: "You are a helpful assistant.", + // }, + // { + // role: "user", + // content: "Hello!", + // }, + // ], + // max_tokens: 4096, + // }); + // console.log("Together response for multiple messages", result.choices[0].message?.content); + // } catch (error) { + // console.error(error); + // } + // }, 20000); + + // it("should log the inputs and outputs for multi turn messages in a single trace", async () => { + // if (!repoId || !togetherApiKey) { + // throw new Error("MAXIM_LOG_REPO_ID and TOGETHER_API_KEY environment variables are required"); + // } + // const logger = await maxim.logger({ id: repoId }); + // if (!logger) { + // throw new Error("Logger is not available"); + // } + // const client = wrapMaximTogetherClient(new Together({ apiKey: togetherApiKey }), logger); + + // try { + // const result = await client.chat.completions.create({ + // model: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", + // max_tokens: 512, + // messages: [ + // { + // role: "user", + // content: [ + // { + // type: "text", + // text: "what are the red things in this image?", + // }, + // { + // type: "image_url", + // image_url: { + // url: "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3e/2024_Solar_Eclipse_Prominences.jpg/720px-2024_Solar_Eclipse_Prominences.jpg", + // }, + // }, + // ], + // }, + // ], + // }); + // console.log("Together response for image prompt", result.choices[0].message?.content); + // } catch (error) { + // console.error(error); + // } + // }, 20000); + + // it("should log the user input image and assistant message for image prompt", async () => { + // if (!repoId || !togetherApiKey) { + // throw new Error("MAXIM_LOG_REPO_ID and TOGETHER_API_KEY environment variables are required"); + // } + // const logger = await maxim.logger({ id: repoId }); + // if (!logger) { + // throw new Error("Logger is not available"); + // } + // const client = wrapMaximTogetherClient(new Together({ apiKey: togetherApiKey }), logger); + + // try { + // const result = await client.chat.completions.create({ + // model: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", + // max_tokens: 512, + // messages: [ + // { + // role: "user", + // content: [ + // { + // type: "text", + // text: "what are the red things in this image?", + // }, + // { + // type: "image_url", + // image_url: { + // url: "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3e/2024_Solar_Eclipse_Prominences.jpg/720px-2024_Solar_Eclipse_Prominences.jpg", + // }, + // }, + // ], + // }, + // ], + // }); + // console.log("Together response for image prompt", result.choices[0].message?.content); + // } catch (error) { + // console.error(error); + // } + // }, 40000); + + // it("should log the user input and the model response for stream text", async () => { + // if (!repoId || !togetherApiKey) { + // throw new Error("MAXIM_LOG_REPO_ID and TOGETHER_API_KEY environment variables are required"); + // } + // const logger = await maxim.logger({ id: repoId }); + // if (!logger) { + // throw new Error("Logger is not available"); + // } + // const client = wrapMaximTogetherClient(new Together({ apiKey: togetherApiKey }), logger); + + // try { + // const result = client.chat.completions.stream({ + // model: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", + // max_tokens: 512, + // temperature: 0.3, + // messages: [ + // { + // role: "user", + // content: "Invent a new holiday and describe its traditions.", + // }, + // ], + // }); + // let fullText = ""; + // for await (const chunk of result) { + // if (chunk.choices?.[0]?.delta?.content) { + // fullText += chunk.choices[0].delta.content; + // } + // } + // console.log("Together response for stream text", fullText); + // } catch (error) { + // console.error(error); + // } + // }, 40000); + + + // it("should log the user input and the model response for stream text with chat prompt", async () => { + // if (!repoId || !togetherApiKey) { + // throw new Error("MAXIM_LOG_REPO_ID and TOGETHER_API_KEY environment variables are required"); + // } + // const logger = await maxim.logger({ id: repoId }); + // if (!logger) { + // throw new Error("Logger is not available"); + // } + // const client = wrapMaximTogetherClient(new Together({ apiKey: togetherApiKey }), logger); + + // try { + // const result = await client.chat.completions.stream({ + // model: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", + // max_tokens: 1024, + // temperature: 0.3, + // messages: [ + // { + // role: "user", + // content: "Hello!", + // }, + // { + // role: "assistant", + // content: "Hello! How can I help you today?", + // }, + // { + // role: "user", + // content: "I need help with my computer.", + // }, + // ], + // }); + // let fullText = ""; + // for await (const chunk of result) { + // if (chunk.choices?.[0]?.delta?.content) { + // fullText += chunk.choices[0].delta.content; + // } + // } + // console.log("Together response for stream text with chat prompt", fullText); + // } catch (error) { + // console.error("Error in stream text with chat prompt", error); + // } + // }, 40000); + + // it("should log the user input and the model response for stream text", async () => { + // if (!repoId || !togetherApiKey) { + // throw new Error("MAXIM_LOG_REPO_ID and TOGETHER_API_KEY environment variables are required"); + // } + // const logger = await maxim.logger({ id: repoId }); + // if (!logger) { + // throw new Error("Logger is not available"); + // } + // const client = wrapMaximTogetherClient(new Together({ apiKey: togetherApiKey }), logger); + + // try { + // const result = await client.chat.completions.stream({ + // model: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", + // max_tokens: 512, + // temperature: 0.3, + // messages: [ + // { + // role: "user", + // content: [ + // { type: "text", text: "Describe the image in detail." }, + // { + // type: "image_url", + // image_url: { + // url: "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3e/2024_Solar_Eclipse_Prominences.jpg/720px-2024_Solar_Eclipse_Prominences.jpg", + // }, + // }, + // ], + // }, + // ], + // }); + // let fullText = ""; + // for await (const chunk of result) { + // if (chunk.choices?.[0]?.delta?.content) { + // fullText += chunk.choices[0].delta.content; + // } + // } + // console.log("Together response for image prompt with streamed text", fullText); + // } catch (error) { + // console.error("Error in image prompt with streamed text", error); + // } + // // }, 20000); + + // it("should log the user input and the model response in response_format", async () => { + // if (!repoId || !togetherApiKey) { + // throw new Error("MAXIM_LOG_REPO_ID and TOGETHER_API_KEY environment variables are required"); + // } + // const logger = await maxim.logger({ id: repoId }); + // if (!logger) { + // throw new Error("Logger is not available"); + // } + // const client = wrapMaximTogetherClient(new Together({ apiKey: togetherApiKey }), logger); + + // try { + // const result = await client.chat.completions.create({ + // model: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", + // max_tokens: 512, + // temperature: 0.3, + // messages: [ + // { + // role: "user", + // content: "Generate a lasagna recipe.", + // }, + // ], + // response_format: { + // type: "json_object", + // }, + // }); + // console.log("Together response for generate object", result.choices[0].message?.content); + // } catch (error) { + // console.error("Error in generate object", error); + // } + // }, 20000); + + // it("should log Together.ai image generation with Maxim attachments", async () => { + // if (!repoId || !togetherApiKey) { + // throw new Error("MAXIM_LOG_REPO_ID and TOGETHER_API_KEY environment variables are required"); + // } + // const logger = await maxim.logger({ id: repoId }); + // if (!logger) { + // throw new Error("Logger is not available"); + // } + // const client = wrapMaximTogetherClient(new Together({ apiKey: togetherApiKey }), logger); + + // try { + // const result = await client.images.create({ + // model: "black-forest-labs/FLUX.1-schnell-Free", + // prompt: "A beautiful sunset over a calm ocean", + // n: 1, + // }); + // console.log("Together image generation result:", result.data[0]); + // } catch (error) { + // console.error("Error in image generation with Maxim logging:", error); + // } + // }, 40000); + + // it("should demonstrate Together.ai chat completion with full Maxim metadata", async () => { + // if (!repoId || !togetherApiKey) { + // throw new Error("MAXIM_LOG_REPO_ID and TOGETHER_API_KEY environment variables are required"); + // } + // const logger = await maxim.logger({ id: repoId }); + // if (!logger) { + // throw new Error("Logger is not available"); + // } + // const client = wrapMaximTogetherClient(new Together({ apiKey: togetherApiKey }), logger); + + // try { + // const result = await client.chat.completions.create({ + // model: "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + // messages: [ + // { + // role: "system", + // content: "You are a helpful assistant that provides concise responses." + // }, + // { + // role: "user", + // content: "What is the capital of France?" + // } + // ], + // temperature: 0.7, + // max_tokens: 100, + // maxim: { + // sessionId: "demo-session-123", + // sessionName: "Together.ai Demo Session", + // sessionTags: { + // environment: "test", + // model_provider: "together", + // demo_type: "metadata_showcase" + // }, + // traceName: "capital-question-trace", + // traceTags: { + // question_type: "geography", + // difficulty: "easy", + // expected_answer: "Paris" + // }, + // generationName: "france-capital-response", + // generationTags: { + // response_type: "factual", + // category: "geography", + // language: "english" + // } + // } + // }); + + // console.log("Together.ai chat completion result:", result.choices[0]?.message?.content); + // console.log("This should appear on Maxim dashboard with:"); + // console.log("- Session: 'Together.ai Demo Session' with environment and provider tags"); + // console.log("- Trace: 'capital-question-trace' with question metadata"); + // console.log("- Generation: 'france-capital-response' with response categorization"); + // } catch (error) { + // console.error("Error in chat completion with Maxim metadata:", error); + // } + // }, 20000); + }); +}); diff --git a/src/lib/logger/together/utils.ts b/src/lib/logger/together/utils.ts new file mode 100644 index 0000000..9ca86da --- /dev/null +++ b/src/lib/logger/together/utils.ts @@ -0,0 +1,80 @@ +import { v4 as uuid } from "uuid"; +import { Generation, Trace } from "../components"; + + +/** + * Processes a Together.ai stream and logs the result to Maxim tracing. + */ +export function processTogetherStream( + stream: any, + trace: Trace, + generation: Generation, + model: string, + maximMetadata: any, +) { + let chunks: any[] = []; + let result = { + text: "", + usage: { prompt_tokens: 0, completion_tokens: 0 }, + finishReason: "stop", + }; + + const processedStream = new ReadableStream({ + async start(controller) { + try { + for await (const chunk of stream) { + chunks.push(chunk); + + // Extract text from chunk + if (chunk.choices?.[0]?.delta?.content) { + result.text += chunk.choices[0].delta.content; + } + + // Update usage if available + if (chunk.usage) { + result.usage = chunk.usage; + } + + // Check for finish reason + if (chunk.choices?.[0]?.finish_reason) { + result.finishReason = chunk.choices[0].finish_reason; + } + + controller.enqueue(chunk); + } + + // Log the final result + generation.result({ + id: uuid(), + object: "chat_completion", + created: Math.floor(Date.now() / 1000), + model: model, + choices: [{ + index: 0, + text: result.text, + finish_reason: result.finishReason, + logprobs: null, + }], + usage: { + prompt_tokens: result.usage.prompt_tokens || 0, + completion_tokens: result.usage.completion_tokens || 0, + total_tokens: (result.usage.prompt_tokens || 0) + (result.usage.completion_tokens || 0), + }, + }); + generation.end(); + + controller.close(); + } catch (error) { + generation.error({ + message: (error as Error).message, + }); + console.error("[Maxim SDK] Together.ai stream logging failed:", error); + controller.error(error); + } finally { + if (!maximMetadata?.traceId) trace.end(); + } + }, + }); + + return processedStream; +} diff --git a/src/lib/logger/together/wrapper.ts b/src/lib/logger/together/wrapper.ts new file mode 100644 index 0000000..c60a1e2 --- /dev/null +++ b/src/lib/logger/together/wrapper.ts @@ -0,0 +1,288 @@ +import { Together } from "together-ai"; +import { MaximLogger } from "../logger"; +import { v4 as uuid } from "uuid"; +import { ChatCompletionResult, Generation, Trace } from "../components"; +import { Attachment } from "../components/attachment"; +import { processTogetherStream } from "./utils"; + +/** + * Metadata options for Maxim tracing integration with Together.ai SDK. + */ +export type MaximTogetherProviderMetadata = { + /** Link your traces to a session */ + sessionId?: string; + /** Override session name */ + sessionName?: string; + /** Add tags to session */ + sessionTags?: Record; + /** Pass in an existing trace's id */ + traceId?: string; + /** Override trace name */ + traceName?: string; + /** Add tags to trace */ + traceTags?: Record; + /** Pass in a custom generation name */ + generationName?: string; + /** Add tags to generation */ + generationTags?: Record; +}; + +/** + * Wraps a Together.ai client with Maxim logging and tracing capabilities. + * + * This function returns a wrapped version of the Together.ai client that integrates + * Maxim's observability features for completions and image generation. + * + * @param client - The Together.ai client instance to wrap. + * @param logger - The MaximLogger instance to use for tracing and logging. + * @param maximMetadata - Optional default Maxim metadata to apply to all requests. + * @returns The wrapped client with Maxim integration. + */ +export function wrapMaximTogetherClient( + client: Together, + logger: MaximLogger, + maximMetadata?: MaximTogetherProviderMetadata +): Together { + return new Proxy(client, { + get(target, prop, receiver) { + if (prop === "chat") { + return { + completions: { + create: async (options: any) => { + let generation: Generation | undefined = undefined; + + try { + const response = await target.chat.completions.create(options); + + let trace: Trace | undefined = undefined; + + if (maximMetadata?.traceId) { + trace = logger.trace({ + id: maximMetadata.traceId, + name: maximMetadata?.traceName ?? "together-completion", + tags: maximMetadata?.traceTags + }); + } else if (maximMetadata?.sessionId) { + const session = logger.session({ + id: maximMetadata.sessionId, + name: maximMetadata?.sessionName ?? "default-session", + tags: maximMetadata?.sessionTags + }); + + trace = session.trace({ + id: uuid(), + name: maximMetadata?.traceName ?? "together-completion", + tags: maximMetadata?.traceTags + }); + } else { + trace = logger.trace({ + id: uuid(), + name: maximMetadata?.traceName ?? "together-completion", + tags: maximMetadata?.traceTags + }); + } + + generation = trace.generation({ + id: uuid(), + name: maximMetadata?.generationName ?? "default-generation", + provider: "together", + model: options.model || "unknown", + messages: options.messages, + modelParameters: options, + tags: maximMetadata?.generationTags, + }); + + generation.result(response as ChatCompletionResult); + generation.end(); + trace.end(); + + return response; + } catch (error) { + if (generation) { + generation.error({ message: (error as Error).message }); + generation.end(); + } + console.error('[MaximSDK] Together.ai chat completion failed:', error); + throw error; + } + }, + stream: (options: any) => { + let generation: Generation | undefined = undefined; + + try { + const response = target.chat.completions.stream(options); + + let trace: Trace | undefined = undefined; + + if (maximMetadata?.traceId) { + trace = logger.trace({ + id: maximMetadata.traceId, + name: maximMetadata?.traceName ?? "together-completion", + tags: maximMetadata?.traceTags + }); + } else if (maximMetadata?.sessionId) { + const session = logger.session({ + id: maximMetadata.sessionId, + name: maximMetadata?.sessionName ?? "default-session", + tags: maximMetadata?.sessionTags + }); + + trace = session.trace({ + id: uuid(), + name: maximMetadata?.traceName ?? "together-completion", + tags: maximMetadata?.traceTags + }); + } else { + trace = logger.trace({ + id: uuid(), + name: maximMetadata?.traceName ?? "together-completion", + tags: maximMetadata?.traceTags + }); + } + + generation = trace.generation({ + id: uuid(), + name: maximMetadata?.generationName ?? "default-generation", + provider: "together", + model: options.model, + messages: options.messages, + modelParameters: options, + tags: maximMetadata?.generationTags, + }); + + const processedStream = processTogetherStream(response, trace, generation, options.model, maximMetadata); + return processedStream; + + } catch (error) { + if (generation) { + generation.error({ + message: (error as Error).message + }); + generation.end(); + } + + console.error('[MaximSDK] Together.ai chat completion stream failed:', error); + throw error; + } + } + } + }; + } + + if (prop === "images") { + return { + create: async (options: any) => { + let generation: Generation | undefined = undefined; + + try { + const response = await target.images.create(options); + + let trace: Trace | undefined = undefined; + + if (maximMetadata?.traceId) { + trace = logger.trace({ + id: maximMetadata.traceId, + name: maximMetadata?.traceName ?? "together-image-generation", + tags: maximMetadata?.traceTags + }); + } else if (maximMetadata?.sessionId) { + const session = logger.session({ + id: maximMetadata.sessionId, + name: maximMetadata?.sessionName ?? "default-session", + tags: maximMetadata?.sessionTags + }); + + trace = session.trace({ + id: uuid(), + name: maximMetadata?.traceName ?? "together-image-generation", + tags: maximMetadata?.traceTags + }); + } else { + trace = logger.trace({ + id: uuid(), + name: maximMetadata?.traceName ?? "together-image-generation", + tags: maximMetadata?.traceTags + }); + } + + generation = trace.generation({ + id: uuid(), + name: maximMetadata?.generationName ?? "default-generation", + provider: "together", + model: options.model || "unknown", + messages: [{ role: "user", content: options.prompt }], + modelParameters: options, + tags: maximMetadata?.generationTags, + }); + + if (response.data && Array.isArray(response.data)) { + response.data.forEach((imageData: any, index: number) => { + let attachment: Attachment; + + if (imageData.url) { + attachment = { + id: uuid(), + type: "url", + url: imageData.url, + name: `generated-image-${index + 1}`, + }; + } else if (imageData.b64_json) { + const base64Data = imageData.b64_json; + const binaryData = Buffer.from(base64Data, 'base64'); + + attachment = { + id: uuid(), + type: "fileData", + data: binaryData, + name: `generated-image-${index + 1}.png`, + }; + } else { + return; + } + + generation?.addAttachment(attachment); + }); + } + + const completionResult: ChatCompletionResult = { + id: uuid(), + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: options.model || "unknown", + choices: [{ + index: 0, + message: { + role: "assistant", + content: `Generated ${response.data?.length || 0} image(s) for prompt: "${options.prompt}"` + }, + logprobs: null, + finish_reason: "stop" + }], + usage: { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0 + } + }; + + generation.result(completionResult); + generation.end(); + trace.end(); + + return response; + } catch (error) { + if (generation) { + generation.error({ message: (error as Error).message }); + generation.end(); + } + console.error('[MaximSDK] Together.ai image generation failed:', error); + throw error; + } + } + }; + } + + return Reflect.get(target, prop, receiver); + } + }) as Together; +} \ No newline at end of file diff --git a/together-ai-sdk.ts b/together-ai-sdk.ts new file mode 100644 index 0000000..82a76a3 --- /dev/null +++ b/together-ai-sdk.ts @@ -0,0 +1,2 @@ +export type { MaximTogetherProviderMetadata } from "./src/lib/logger/together/wrapper"; +export { wrapMaximTogetherClient } from "./src/lib/logger/together/wrapper"; \ No newline at end of file