From 04c93320e4ddeb3a3b0d7a7c358cbc3f0462bef6 Mon Sep 17 00:00:00 2001 From: Sakshi3027 Date: Sun, 14 Jun 2026 12:44:41 -0400 Subject: [PATCH] feat: add multi-model comparison benchmark for Crusoe Managed Inference --- model-comparison-crusoe/.gitignore | 4 + model-comparison-crusoe/README.md | 86 +++++++++++++++ model-comparison-crusoe/compare.py | 127 +++++++++++++++++++++++ model-comparison-crusoe/requirements.txt | 4 + 4 files changed, 221 insertions(+) create mode 100644 model-comparison-crusoe/.gitignore create mode 100644 model-comparison-crusoe/README.md create mode 100644 model-comparison-crusoe/compare.py create mode 100644 model-comparison-crusoe/requirements.txt diff --git a/model-comparison-crusoe/.gitignore b/model-comparison-crusoe/.gitignore new file mode 100644 index 0000000..0a16c91 --- /dev/null +++ b/model-comparison-crusoe/.gitignore @@ -0,0 +1,4 @@ +.env +__pycache__/ +*.pyc +.DS_Store diff --git a/model-comparison-crusoe/README.md b/model-comparison-crusoe/README.md new file mode 100644 index 0000000..f0306e8 --- /dev/null +++ b/model-comparison-crusoe/README.md @@ -0,0 +1,86 @@ +# Multi-Model Comparison × Crusoe AI + +Benchmark multiple models on [Crusoe Managed Inference](https://www.crusoe.ai/cloud/managed-inference) across reasoning, code generation, and summarization — with latency, word count, and throughput metrics. + +## What this does + +Runs 3 models × 3 tasks concurrently and produces a leaderboard: +TASK: Reasoning + +Llama-3.3-70B-Instruct 1.2s | 101 words | 84.2 wps + +DeepSeek-V3-0324 0.9s | 118 words | 131.1 wps + +Qwen3-235B-A22B 1.4s | 142 words | 101.4 wps +LEADERBOARD (average latency across all tasks) + +1st DeepSeek-V3-0324 avg 0.95s | 128.3 wps + +2nd Llama-3.3-70B avg 1.18s | 97.2 wps + +3rd Qwen3-235B avg 1.41s | 108.7 wps + +## Prerequisites + +- Python 3.10+ +- A Crusoe Cloud account → [console.crusoecloud.com](https://console.crusoecloud.com) +- Inference API key (Intelligence API Keys section under Security) + +## Setup + +```bash +pip install -r requirements.txt +export CRUSOE_API_KEY="your-api-key" +``` + +## Run the benchmark + +```bash +python compare.py +``` + +## Local testing (no Crusoe account needed) + +```bash +pip install langchain-groq +export GROQ_API_KEY="your-groq-key" # free at console.groq.com +python compare.py +``` + +## Models compared on Crusoe + +| Model | Provider | Context | +|-------|----------|---------| +| `meta-llama/Llama-3.3-70B-Instruct` | Meta | 128k | +| `deepseek-ai/DeepSeek-V3-0324` | DeepSeek | 160k | +| `Qwen/Qwen3-235B-A22B` | Qwen | 131k | + +See the full model list at [Crusoe Intelligence Foundry](https://console.crusoecloud.com/foundry/models). + +## Tasks + +| Task | What it tests | +|------|--------------| +| Reasoning | Multi-step math problem | +| Code generation | Sieve of Eratosthenes with type hints | +| Summarization | SQL vs NoSQL tradeoffs in 3 bullets | + +## Add your own tasks + +```python +TASKS = [ + { + "name": "My task", + "prompt": "Your prompt here", + }, +] +``` + +Each task runs concurrently across all models — total time equals the slowest single call. + +## Related + +- [langchain-crusoe](../langchain-crusoe/) — LangChain integration for Crusoe Managed Inference +- [async-batch-crusoe](../async-batch-crusoe/) — Async batch inference benchmark +- [streaming-crusoe](../streaming-crusoe/) — Real-time token streaming on Crusoe +- [Crusoe Managed Inference Docs](https://docs.crusoecloud.com/managed-inference/overview) diff --git a/model-comparison-crusoe/compare.py b/model-comparison-crusoe/compare.py new file mode 100644 index 0000000..1ad0095 --- /dev/null +++ b/model-comparison-crusoe/compare.py @@ -0,0 +1,127 @@ +""" +Multi-model comparison on Crusoe Managed Inference. +Benchmark multiple models across quality, latency, and throughput. +Tested locally with Groq as a drop-in replacement for Crusoe. +""" +import os +import time +import asyncio +from dotenv import load_dotenv +from langchain_core.messages import HumanMessage + +load_dotenv() + +# Models available on Crusoe Managed Inference +CRUSOE_MODELS = [ + "meta-llama/Llama-3.3-70B-Instruct", + "deepseek-ai/DeepSeek-V3-0324", + "Qwen/Qwen3-235B-A22B", +] + +# Free Groq models for local testing (all verified active) +GROQ_MODELS = [ + "llama-3.3-70b-versatile", + "llama-3.1-8b-instant", + "meta-llama/llama-4-scout-17b-16e-instruct", +] + +TASKS = [ + { + "name": "Reasoning", + "prompt": "If a train travels 120 miles in 2 hours, then stops for 30 minutes, then travels 90 miles in 1.5 hours, what is the average speed for the entire journey including the stop?", + }, + { + "name": "Code generation", + "prompt": "Write a Python function that finds all prime numbers up to n using the Sieve of Eratosthenes. Include type hints and a docstring.", + }, + { + "name": "Summarization", + "prompt": "Summarize the key tradeoffs between SQL and NoSQL databases in exactly 3 bullet points.", + }, +] + + +def get_llm(model: str): + if os.getenv("CRUSOE_API_KEY"): + from langchain_crusoe import ChatCrusoe + return ChatCrusoe(model=model, temperature=0.3, max_tokens=512) + else: + from langchain_groq import ChatGroq + return ChatGroq(model=model, temperature=0.3, max_tokens=512) + + +def get_models(): + if os.getenv("CRUSOE_API_KEY"): + return CRUSOE_MODELS + return GROQ_MODELS + + +async def run_task(model: str, task: dict) -> dict: + """Run a single task on a single model and return metrics.""" + llm = get_llm(model) + start = time.time() + response = await llm.ainvoke([HumanMessage(content=task["prompt"])]) + elapsed = time.time() - start + output = response.content + word_count = len(output.split()) + return { + "model": model.split("/")[-1], + "task": task["name"], + "latency": round(elapsed, 2), + "words": word_count, + "wps": round(word_count / elapsed, 1), + "output": output, + } + + +async def run_all_comparisons(): + models = get_models() + print("Multi-Model Comparison on Crusoe Managed Inference") + print("=" * 60) + print(f"Models: {len(models)} | Tasks: {len(TASKS)}\n") + + jobs = [ + run_task(model, task) + for model in models + for task in TASKS + ] + results = await asyncio.gather(*jobs) + + # Results by task + for task in TASKS: + print(f"\nTASK: {task['name']}") + print("-" * 60) + task_results = [r for r in results if r["task"] == task["name"]] + task_results.sort(key=lambda x: x["latency"]) + for r in task_results: + print(f" {r['model']:<45} {r['latency']}s | {r['words']} words | {r['wps']} wps") + + # Leaderboard + print("\n" + "=" * 60) + print("LEADERBOARD (average latency across all tasks)") + print("=" * 60) + model_names = list({r["model"] for r in results}) + leaderboard = [] + for name in model_names: + model_results = [r for r in results if r["model"] == name] + avg_latency = round(sum(r["latency"] for r in model_results) / len(model_results), 2) + avg_wps = round(sum(r["wps"] for r in model_results) / len(model_results), 1) + leaderboard.append((name, avg_latency, avg_wps)) + leaderboard.sort(key=lambda x: x[1]) + + for i, (name, lat, wps) in enumerate(leaderboard): + medal = ["1st", "2nd", "3rd"][i] if i < 3 else f"{i+1}th" + print(f" {medal} {name:<43} avg {lat}s | {wps} wps") + + # Sample outputs + print("\n" + "=" * 60) + print("SAMPLE OUTPUTS: Code generation") + print("=" * 60) + code_results = [r for r in results if r["task"] == "Code generation"] + for r in code_results: + print(f"\n--- {r['model']} ---") + print(r["output"][:400] + "..." if len(r["output"]) > 400 else r["output"]) + + +if __name__ == "__main__": + asyncio.run(run_all_comparisons()) diff --git a/model-comparison-crusoe/requirements.txt b/model-comparison-crusoe/requirements.txt new file mode 100644 index 0000000..59cc96a --- /dev/null +++ b/model-comparison-crusoe/requirements.txt @@ -0,0 +1,4 @@ +langchain-crusoe>=0.1.0 +langchain-groq>=1.1.0 +langchain-core>=1.0.0 +python-dotenv