diff --git a/README.md b/README.md index 68d076e..a5b3a79 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ PromptLens runs golden test sets against multiple models, scores outputs using L - **Multiple Export Formats** - HTML, JSON, CSV, and Markdown outputs - **Parallel Execution** - Async execution with configurable concurrency and retry logic - **Portable & Local** - No cloud backend, all data stays on your machine +- **Safer HTTP Integrations** - Unparseable HTTP provider responses fail fast with clear error context - **Easy to Extend** - Plugin architecture for custom providers, judges, and exporters --- diff --git a/promptlens/providers/http.py b/promptlens/providers/http.py index f1828e2..1bbaab9 100644 --- a/promptlens/providers/http.py +++ b/promptlens/providers/http.py @@ -123,6 +123,11 @@ async def _make_request() -> ModelResponse: # Extract content (try common response formats) content = self._extract_content(data) + if not content.strip(): + raise ValueError( + "HTTP provider response did not contain parseable text content. " + f"Top-level keys: {sorted(data.keys()) if isinstance(data, dict) else type(data).__name__}" + ) # Local models typically don't provide token counts or cost return ModelResponse( diff --git a/tests/test_http_provider_empty_content_guard.py b/tests/test_http_provider_empty_content_guard.py new file mode 100644 index 0000000..19199e8 --- /dev/null +++ b/tests/test_http_provider_empty_content_guard.py @@ -0,0 +1,51 @@ +from unittest.mock import patch + +import pytest + +from promptlens.models.config import ProviderConfig +from promptlens.providers.http import HTTPProvider + + +def _provider() -> HTTPProvider: + return HTTPProvider( + ProviderConfig( + name="http", + model="test-model", + endpoint="http://localhost:11434/api/generate", + ) + ) + + +@pytest.mark.asyncio +async def test_generate_returns_error_when_content_unparseable() -> None: + provider = _provider() + + class _FakeResponse: + def raise_for_status(self) -> None: + return None + + async def json(self): + return {"foo": "bar"} + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return None + + class _FakeSession: + def post(self, *args, **kwargs): + return _FakeResponse() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return None + + with patch("promptlens.providers.http.aiohttp.ClientSession", return_value=_FakeSession()): + result = await provider.generate("hello") + + assert result.error is not None + assert "did not contain parseable text content" in result.error + assert result.content == ""