feat: add LiteLLM as AI gateway provider

src/openbench/_registry.py

@@ -107,6 +107,14 @@ def deepinfra() -> Type[ModelAPI]:
     return DeepInfraAPI
 
 
+@modelapi(name="litellm")
+def litellm() -> Type[ModelAPI]:
+    """Register LiteLLM AI gateway provider."""
+    from .model._providers.litellm import LiteLLMAPI
+
+    return LiteLLMAPI
+
+
 @modelapi(name="ai21")
 def ai21() -> Type[ModelAPI]:
     """Register AI21 Labs provider."""

src/openbench/model/_providers/litellm.py

@@ -0,0 +1,50 @@
+"""LiteLLM AI gateway provider implementation."""
+
+import os
+from typing import Any
+
+from inspect_ai.model._providers.openai_compatible import OpenAICompatibleAPI
+from inspect_ai.model import GenerateConfig
+
+
+class LiteLLMAPI(OpenAICompatibleAPI):
+    """LiteLLM AI gateway provider - access 100+ LLM providers through one interface.
+
+    Uses OpenAI-compatible API pointed at a LiteLLM proxy. The model name
+    is passed through as-is, so use whatever model identifiers your LiteLLM
+    config defines (e.g. ``litellm/anthropic/claude-sonnet-4-6``).
+    """
+
+    def __init__(
+        self,
+        model_name: str,
+        base_url: str | None = None,
+        api_key: str | None = None,
+        config: GenerateConfig = GenerateConfig(),
+        **model_args: Any,
+    ) -> None:
+        model_name_clean = model_name.replace("litellm/", "", 1)
+
+        base_url = base_url or os.environ.get(
+            "LITELLM_BASE_URL", "http://localhost:4000/v1"
+        )
+        api_key = api_key or os.environ.get("LITELLM_API_KEY")
+
+        if not api_key:
+            raise ValueError(
+                "LiteLLM API key not found. Set LITELLM_API_KEY environment variable."
+            )
+
+        super().__init__(
+            model_name=model_name_clean,
+            base_url=base_url,
+            api_key=api_key,
+            config=config,
+            service="litellm",
+            service_base_url="http://localhost:4000/v1",
+            **model_args,
+        )
+
+    def service_model_name(self) -> str:
+        """Return model name without service prefix."""
+        return self.model_name

src/openbench/provider_config.py

@@ -31,6 +31,7 @@ class ProviderType(str, Enum):
     HUGGINGFACE = "huggingface"
     HYPERBOLIC = "hyperbolic"
     LAMBDA = "lambda"
+    LITELLM = "litellm"
     MINIMAX = "minimax"
     MISTRAL = "mistral"
     MOONSHOT = "moonshot"
@@ -232,6 +233,15 @@ def get_all_env_vars(self) -> List[str]:
         supports_vision=False,
         supports_function_calling=True,
     ),
+    ProviderType.LITELLM: ProviderConfig(
+        name="litellm",
+        display_name="LiteLLM AI Gateway",
+        api_key_env="LITELLM_API_KEY",
+        base_url="http://localhost:4000/v1",
+        base_url_env="LITELLM_BASE_URL",
+        supports_vision=True,
+        supports_function_calling=True,
+    ),
     ProviderType.MINIMAX: ProviderConfig(
         name="minimax",
         display_name="MiniMax",

tests/test_litellm_provider.py

@@ -0,0 +1,81 @@
+"""Unit tests for LiteLLM AI gateway provider."""
+
+import os
+from unittest.mock import patch
+
+import pytest
+
+from openbench.model._providers.litellm import LiteLLMAPI
+
+
+class TestLiteLLMProvider:
+    """Test LiteLLM provider initialization and configuration."""
+
+    def test_raises_without_api_key(self):
+        """Provider raises ValueError when no API key is available."""
+        with patch.dict(os.environ, {}, clear=True):
+            with pytest.raises(ValueError, match="LITELLM_API_KEY"):
+                with patch("inspect_ai.model._providers.openai_compatible.AsyncOpenAI"):
+                    LiteLLMAPI(model_name="litellm/gpt-4o")
+
+    def test_strips_service_prefix_from_model_name(self):
+        """The litellm/ prefix is stripped before passing to the base class."""
+        with patch.dict(os.environ, {"LITELLM_API_KEY": "sk-test"}):
+            with patch("inspect_ai.model._providers.openai_compatible.AsyncOpenAI"):
+                provider = LiteLLMAPI(
+                    model_name="litellm/anthropic/claude-sonnet-4-6",
+                    api_key="sk-test",
+                    base_url="http://localhost:4000/v1",
+                )
+                assert provider.model_name == "anthropic/claude-sonnet-4-6"
+
+    def test_uses_env_api_key(self):
+        """Provider reads LITELLM_API_KEY from environment."""
+        with patch.dict(os.environ, {"LITELLM_API_KEY": "sk-from-env"}):
+            with patch("inspect_ai.model._providers.openai_compatible.AsyncOpenAI"):
+                provider = LiteLLMAPI(
+                    model_name="litellm/gpt-4o",
+                    base_url="http://localhost:4000/v1",
+                )
+                assert provider.api_key == "sk-from-env"
+
+    def test_explicit_api_key_overrides_env(self):
+        """Explicit api_key parameter takes precedence over env var."""
+        with patch.dict(os.environ, {"LITELLM_API_KEY": "sk-from-env"}):
+            with patch("inspect_ai.model._providers.openai_compatible.AsyncOpenAI"):
+                provider = LiteLLMAPI(
+                    model_name="litellm/gpt-4o",
+                    api_key="sk-explicit",
+                    base_url="http://localhost:4000/v1",
+                )
+                assert provider.api_key == "sk-explicit"
+
+    def test_default_base_url(self):
+        """Provider defaults to localhost:4000 when no base URL is set."""
+        with patch.dict(os.environ, {"LITELLM_API_KEY": "sk-test"}, clear=True):
+            with patch("inspect_ai.model._providers.openai_compatible.AsyncOpenAI"):
+                provider = LiteLLMAPI(model_name="litellm/gpt-4o")
+                assert provider.base_url == "http://localhost:4000/v1"
+
+    def test_custom_base_url_from_env(self):
+        """Provider reads LITELLM_BASE_URL from environment."""
+        with patch.dict(
+            os.environ,
+            {
+                "LITELLM_API_KEY": "sk-test",
+                "LITELLM_BASE_URL": "https://proxy.example.com/v1",
+            },
+        ):
+            with patch("inspect_ai.model._providers.openai_compatible.AsyncOpenAI"):
+                provider = LiteLLMAPI(model_name="litellm/gpt-4o")
+                assert provider.base_url == "https://proxy.example.com/v1"
+
+    def test_service_model_name_returns_clean_name(self):
+        """service_model_name() returns the model name without prefix."""
+        with patch.dict(os.environ, {"LITELLM_API_KEY": "sk-test"}):
+            with patch("inspect_ai.model._providers.openai_compatible.AsyncOpenAI"):
+                provider = LiteLLMAPI(
+                    model_name="litellm/openai/gpt-4o",
+                    base_url="http://localhost:4000/v1",
+                )
+                assert provider.service_model_name() == "openai/gpt-4o"