Add multi-LLM support and update dependencies in README and agent.py

README.md

@@ -66,6 +66,60 @@ rm -rf jobs; mkdir -p jobs && uv run harbor run -p tasks/ --task-name "<task-nam
 rm -rf jobs; mkdir -p jobs && uv run harbor run -p tasks/ -n 100 --agent-import-path agent:AutoAgent -o jobs --job-name latest > run.log 2>&1
 ```
 
+## Multi-LLM Support
+
+The harness supports multiple LLM providers via [LiteLLM](https://github.com/BerriAI/litellm). Configure via environment variables:
+
+### Environment Variables
+
+- `LLM_PROVIDER`: Provider name (`openai`, `anthropic`, `ollama`, `azure`, etc.)
+- `MODEL`: Model name (e.g., `gpt-5`, `claude-3-5-sonnet`, `qwen3.5:35b-a3b-q8_0`)
+- `LLM_BASE_URL`: Optional base URL (required for Ollama, Azure, etc.)
+- `API_KEY`: Provider-specific API key (e.g., `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`)
+
+### Using Local Ollama
+
+```bash
+cat > .env << 'EOF'
+LLM_PROVIDER=ollama
+MODEL=qwen3.5:35b-a3b-q8_0
+LLM_BASE_URL=http://host.docker.internal:11434/v1
+EOF
+```
+
+### Using OpenAI
+
+```bash
+cat > .env << 'EOF'
+LLM_PROVIDER=openai
+MODEL=gpt-5
+OPENAI_API_KEY=your-api-key
+EOF
+```
+
+### Using Anthropic
+
+```bash
+cat > .env << 'EOF'
+LLM_PROVIDER=anthropic
+MODEL=claude-3-5-sonnet
+ANTHROPIC_API_KEY=your-api-key
+EOF
+```
+
+### Using Azure
+
+```bash
+cat > .env << 'EOF'
+LLM_PROVIDER=azure
+MODEL=your-deployment-name
+AZURE_API_KEY=your-api-key
+AZURE_API_BASE=https://your-resource.openai.azure.com
+EOF
+```
+
+The model selection is optional and can be changed dynamically by modifying the environment variables before running the benchmark.
+
 ## Running the meta-agent
 
 Point your coding agent at the repo and prompt:

agent.py

@@ -26,8 +26,18 @@
 # ============================================================================
 
 SYSTEM_PROMPT = "You are an agent that executes tasks"
-MODEL = "gpt-5"
-MAX_TURNS = 30
+MAX_TURNS = 15
+
+# Multi-LLM configuration via LiteLLM
+# Set environment variables before running:
+# - LLM_PROVIDER: "openai", "anthropic", "ollama", "azure", etc.
+# - MODEL: model name (e.g., "gpt-5", "claude-3-5-sonnet", "qwen3.5:35b-a3b-q8_0")
+# - LLM_BASE_URL: optional base URL (required for Ollama, Azure, etc.)
+# - API_KEY: provider-specific API key (e.g., OPENAI_API_KEY, ANTHROPIC_API_KEY)
+import os
+LLM_PROVIDER = os.getenv("LLM_PROVIDER", "openai").lower()
+MODEL = os.getenv("MODEL", "gpt-5")
+LLM_BASE_URL = os.getenv("LLM_BASE_URL")
 
 
 def create_tools(environment: BaseEnvironment) -> list[FunctionTool]:
@@ -53,11 +63,25 @@ async def run_shell(command: str) -> str:
 def create_agent(environment: BaseEnvironment) -> Agent:
     """Build the agent. Modify to add handoffs, sub-agents, or agent-as-tool."""
     tools = create_tools(environment)
+
+    # Build LiteLLM-compatible model string
+    if LLM_PROVIDER == "ollama":
+        # Ollama uses custom base URL format
+        model_string = f"ollama_chat/{MODEL}" if not LLM_BASE_URL else f"ollama_chat/{MODEL}"
+    elif LLM_PROVIDER == "azure":
+        # Azure uses deployment name format
+        model_string = f"azure/{MODEL}"
+    elif LLM_PROVIDER == "anthropic":
+        model_string = f"anthropic/{MODEL}"
+    else:
+        # Default to OpenAI format
+        model_string = MODEL
+
     return Agent(
         name="autoagent",
         instructions=SYSTEM_PROMPT,
         tools=tools,
-        model=MODEL,
+        model=model_string,
     )
 
 

pyproject.toml

@@ -9,4 +9,5 @@ dependencies = [
     "openpyxl",
     "numpy",
     "harbor",
+    "litellm>=1.60.0",
 ]