feat: multi-model comparison benchmark for Crusoe Managed Inference

model-comparison-crusoe/.gitignore

@@ -0,0 +1,4 @@
+.env
+__pycache__/
+*.pyc
+.DS_Store

model-comparison-crusoe/README.md

@@ -0,0 +1,86 @@
+# Multi-Model Comparison × Crusoe AI
+
+Benchmark multiple models on [Crusoe Managed Inference](https://www.crusoe.ai/cloud/managed-inference) across reasoning, code generation, and summarization — with latency, word count, and throughput metrics.
+
+## What this does
+
+Runs 3 models × 3 tasks concurrently and produces a leaderboard:
+TASK: Reasoning
+
+Llama-3.3-70B-Instruct    1.2s | 101 words | 84.2 wps
+
+DeepSeek-V3-0324          0.9s | 118 words | 131.1 wps
+
+Qwen3-235B-A22B           1.4s | 142 words | 101.4 wps
+LEADERBOARD (average latency across all tasks)
+
+1st  DeepSeek-V3-0324     avg 0.95s | 128.3 wps
+
+2nd  Llama-3.3-70B        avg 1.18s | 97.2 wps
+
+3rd  Qwen3-235B           avg 1.41s | 108.7 wps
+
+## Prerequisites
+
+- Python 3.10+
+- A Crusoe Cloud account → [console.crusoecloud.com](https://console.crusoecloud.com)
+- Inference API key (Intelligence API Keys section under Security)
+
+## Setup
+
+```bash
+pip install -r requirements.txt
+export CRUSOE_API_KEY="your-api-key"
+```
+
+## Run the benchmark
+
+```bash
+python compare.py
+```
+
+## Local testing (no Crusoe account needed)
+
+```bash
+pip install langchain-groq
+export GROQ_API_KEY="your-groq-key"  # free at console.groq.com
+python compare.py
+```
+
+## Models compared on Crusoe
+
+| Model | Provider | Context |
+|-------|----------|---------|
+| `meta-llama/Llama-3.3-70B-Instruct` | Meta | 128k |
+| `deepseek-ai/DeepSeek-V3-0324` | DeepSeek | 160k |
+| `Qwen/Qwen3-235B-A22B` | Qwen | 131k |
+
+See the full model list at [Crusoe Intelligence Foundry](https://console.crusoecloud.com/foundry/models).
+
+## Tasks
+
+| Task | What it tests |
+|------|--------------|
+| Reasoning | Multi-step math problem |
+| Code generation | Sieve of Eratosthenes with type hints |
+| Summarization | SQL vs NoSQL tradeoffs in 3 bullets |
+
+## Add your own tasks
+
+```python
+TASKS = [
+    {
+        "name": "My task",
+        "prompt": "Your prompt here",
+    },
+]
+```
+
+Each task runs concurrently across all models — total time equals the slowest single call.
+
+## Related
+
+- [langchain-crusoe](../langchain-crusoe/) — LangChain integration for Crusoe Managed Inference
+- [async-batch-crusoe](../async-batch-crusoe/) — Async batch inference benchmark
+- [streaming-crusoe](../streaming-crusoe/) — Real-time token streaming on Crusoe
+- [Crusoe Managed Inference Docs](https://docs.crusoecloud.com/managed-inference/overview)

model-comparison-crusoe/compare.py

@@ -0,0 +1,127 @@
+"""
+Multi-model comparison on Crusoe Managed Inference.
+Benchmark multiple models across quality, latency, and throughput.
+Tested locally with Groq as a drop-in replacement for Crusoe.
+"""
+import os
+import time
+import asyncio
+from dotenv import load_dotenv
+from langchain_core.messages import HumanMessage
+
+load_dotenv()
+
+# Models available on Crusoe Managed Inference
+CRUSOE_MODELS = [
+    "meta-llama/Llama-3.3-70B-Instruct",
+    "deepseek-ai/DeepSeek-V3-0324",
+    "Qwen/Qwen3-235B-A22B",
+]
+
+# Free Groq models for local testing (all verified active)
+GROQ_MODELS = [
+    "llama-3.3-70b-versatile",
+    "llama-3.1-8b-instant",
+    "meta-llama/llama-4-scout-17b-16e-instruct",
+]
+
+TASKS = [
+    {
+        "name": "Reasoning",
+        "prompt": "If a train travels 120 miles in 2 hours, then stops for 30 minutes, then travels 90 miles in 1.5 hours, what is the average speed for the entire journey including the stop?",
+    },
+    {
+        "name": "Code generation",
+        "prompt": "Write a Python function that finds all prime numbers up to n using the Sieve of Eratosthenes. Include type hints and a docstring.",
+    },
+    {
+        "name": "Summarization",
+        "prompt": "Summarize the key tradeoffs between SQL and NoSQL databases in exactly 3 bullet points.",
+    },
+]
+
+
+def get_llm(model: str):
+    if os.getenv("CRUSOE_API_KEY"):
+        from langchain_crusoe import ChatCrusoe
+        return ChatCrusoe(model=model, temperature=0.3, max_tokens=512)
+    else:
+        from langchain_groq import ChatGroq
+        return ChatGroq(model=model, temperature=0.3, max_tokens=512)
+
+
+def get_models():
+    if os.getenv("CRUSOE_API_KEY"):
+        return CRUSOE_MODELS
+    return GROQ_MODELS
+
+
+async def run_task(model: str, task: dict) -> dict:
+    """Run a single task on a single model and return metrics."""
+    llm = get_llm(model)
+    start = time.time()
+    response = await llm.ainvoke([HumanMessage(content=task["prompt"])])
+    elapsed = time.time() - start
+    output = response.content
+    word_count = len(output.split())
+    return {
+        "model": model.split("/")[-1],
+        "task": task["name"],
+        "latency": round(elapsed, 2),
+        "words": word_count,
+        "wps": round(word_count / elapsed, 1),
+        "output": output,
+    }
+
+
+async def run_all_comparisons():
+    models = get_models()
+    print("Multi-Model Comparison on Crusoe Managed Inference")
+    print("=" * 60)
+    print(f"Models: {len(models)} | Tasks: {len(TASKS)}\n")
+
+    jobs = [
+        run_task(model, task)
+        for model in models
+        for task in TASKS
+    ]
+    results = await asyncio.gather(*jobs)
+
+    # Results by task
+    for task in TASKS:
+        print(f"\nTASK: {task['name']}")
+        print("-" * 60)
+        task_results = [r for r in results if r["task"] == task["name"]]
+        task_results.sort(key=lambda x: x["latency"])
+        for r in task_results:
+            print(f"  {r['model']:<45} {r['latency']}s | {r['words']} words | {r['wps']} wps")
+
+    # Leaderboard
+    print("\n" + "=" * 60)
+    print("LEADERBOARD (average latency across all tasks)")
+    print("=" * 60)
+    model_names = list({r["model"] for r in results})
+    leaderboard = []
+    for name in model_names:
+        model_results = [r for r in results if r["model"] == name]
+        avg_latency = round(sum(r["latency"] for r in model_results) / len(model_results), 2)
+        avg_wps = round(sum(r["wps"] for r in model_results) / len(model_results), 1)
+        leaderboard.append((name, avg_latency, avg_wps))
+    leaderboard.sort(key=lambda x: x[1])
+
+    for i, (name, lat, wps) in enumerate(leaderboard):
+        medal = ["1st", "2nd", "3rd"][i] if i < 3 else f"{i+1}th"
+        print(f"  {medal}  {name:<43} avg {lat}s | {wps} wps")
+
+    # Sample outputs
+    print("\n" + "=" * 60)
+    print("SAMPLE OUTPUTS: Code generation")
+    print("=" * 60)
+    code_results = [r for r in results if r["task"] == "Code generation"]
+    for r in code_results:
+        print(f"\n--- {r['model']} ---")
+        print(r["output"][:400] + "..." if len(r["output"]) > 400 else r["output"])
+
+
+if __name__ == "__main__":
+    asyncio.run(run_all_comparisons())

model-comparison-crusoe/requirements.txt

@@ -0,0 +1,4 @@
+langchain-crusoe>=0.1.0
+langchain-groq>=1.1.0
+langchain-core>=1.0.0
+python-dotenv