diff --git a/README.md b/README.md index 1d7fcd11..924d373e 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,60 @@ rm -rf jobs; mkdir -p jobs && uv run harbor run -p tasks/ --task-name " run.log 2>&1 ``` +## Multi-LLM Support + +The harness supports multiple LLM providers via [LiteLLM](https://github.com/BerriAI/litellm). Configure via environment variables: + +### Environment Variables + +- `LLM_PROVIDER`: Provider name (`openai`, `anthropic`, `ollama`, `azure`, etc.) +- `MODEL`: Model name (e.g., `gpt-5`, `claude-3-5-sonnet`, `qwen3.5:35b-a3b-q8_0`) +- `LLM_BASE_URL`: Optional base URL (required for Ollama, Azure, etc.) +- `API_KEY`: Provider-specific API key (e.g., `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`) + +### Using Local Ollama + +```bash +cat > .env << 'EOF' +LLM_PROVIDER=ollama +MODEL=qwen3.5:35b-a3b-q8_0 +LLM_BASE_URL=http://host.docker.internal:11434/v1 +EOF +``` + +### Using OpenAI + +```bash +cat > .env << 'EOF' +LLM_PROVIDER=openai +MODEL=gpt-5 +OPENAI_API_KEY=your-api-key +EOF +``` + +### Using Anthropic + +```bash +cat > .env << 'EOF' +LLM_PROVIDER=anthropic +MODEL=claude-3-5-sonnet +ANTHROPIC_API_KEY=your-api-key +EOF +``` + +### Using Azure + +```bash +cat > .env << 'EOF' +LLM_PROVIDER=azure +MODEL=your-deployment-name +AZURE_API_KEY=your-api-key +AZURE_API_BASE=https://your-resource.openai.azure.com +EOF +``` + +The model selection is optional and can be changed dynamically by modifying the environment variables before running the benchmark. + ## Running the meta-agent Point your coding agent at the repo and prompt: diff --git a/agent.py b/agent.py index d155db41..b104b6bd 100644 --- a/agent.py +++ b/agent.py @@ -26,8 +26,18 @@ # ============================================================================ SYSTEM_PROMPT = "You are an agent that executes tasks" -MODEL = "gpt-5" -MAX_TURNS = 30 +MAX_TURNS = 15 + +# Multi-LLM configuration via LiteLLM +# Set environment variables before running: +# - LLM_PROVIDER: "openai", "anthropic", "ollama", "azure", etc. +# - MODEL: model name (e.g., "gpt-5", "claude-3-5-sonnet", "qwen3.5:35b-a3b-q8_0") +# - LLM_BASE_URL: optional base URL (required for Ollama, Azure, etc.) +# - API_KEY: provider-specific API key (e.g., OPENAI_API_KEY, ANTHROPIC_API_KEY) +import os +LLM_PROVIDER = os.getenv("LLM_PROVIDER", "openai").lower() +MODEL = os.getenv("MODEL", "gpt-5") +LLM_BASE_URL = os.getenv("LLM_BASE_URL") def create_tools(environment: BaseEnvironment) -> list[FunctionTool]: @@ -53,11 +63,25 @@ async def run_shell(command: str) -> str: def create_agent(environment: BaseEnvironment) -> Agent: """Build the agent. Modify to add handoffs, sub-agents, or agent-as-tool.""" tools = create_tools(environment) + + # Build LiteLLM-compatible model string + if LLM_PROVIDER == "ollama": + # Ollama uses custom base URL format + model_string = f"ollama_chat/{MODEL}" if not LLM_BASE_URL else f"ollama_chat/{MODEL}" + elif LLM_PROVIDER == "azure": + # Azure uses deployment name format + model_string = f"azure/{MODEL}" + elif LLM_PROVIDER == "anthropic": + model_string = f"anthropic/{MODEL}" + else: + # Default to OpenAI format + model_string = MODEL + return Agent( name="autoagent", instructions=SYSTEM_PROMPT, tools=tools, - model=MODEL, + model=model_string, ) diff --git a/pyproject.toml b/pyproject.toml index c419b630..df8ceb72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,4 +9,5 @@ dependencies = [ "openpyxl", "numpy", "harbor", + "litellm>=1.60.0", ]