-
Notifications
You must be signed in to change notification settings - Fork 48
Add pyreqwest-backed transports #1088
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| from __future__ import annotations | ||
|
|
||
| import typing | ||
| from collections.abc import AsyncIterator, Iterator | ||
| from contextlib import AsyncExitStack, ExitStack | ||
|
|
||
| from .._models import Request, Response | ||
| from .._types import AsyncByteStream, SyncByteStream | ||
| from .base import AsyncBaseTransport, BaseTransport | ||
|
|
||
| if typing.TYPE_CHECKING: # pragma: no cover | ||
| from pyreqwest.client import Client as PyreqwestClient, SyncClient as PyreqwestSyncClient | ||
| from pyreqwest.response import Response as PyreqwestResponse, SyncResponse as PyreqwestSyncResponse | ||
|
|
||
| __all__ = ["AsyncPyreqwestTransport", "PyreqwestTransport"] | ||
|
|
||
|
|
||
| def _load_pyreqwest_sync_client_builder() -> typing.Any: | ||
| try: | ||
| from pyreqwest.client import SyncClientBuilder | ||
| except ImportError as exc: # pragma: no cover | ||
| msg = "Using 'PyreqwestTransport' requires installing the 'pyreqwest' package." | ||
| raise RuntimeError(msg) from exc | ||
| return SyncClientBuilder | ||
|
|
||
|
|
||
| def _load_pyreqwest_async_client_builder() -> typing.Any: | ||
| try: | ||
| from pyreqwest.client import ClientBuilder | ||
| except ImportError as exc: # pragma: no cover | ||
| msg = "Using 'AsyncPyreqwestTransport' requires installing the 'pyreqwest' package." | ||
| raise RuntimeError(msg) from exc | ||
| return ClientBuilder | ||
|
|
||
|
|
||
| def _http_version(version: str) -> bytes: | ||
| if version == "HTTP/2.0": | ||
| version = "HTTP/2" | ||
| return version.encode("ascii", errors="ignore") | ||
|
|
||
|
|
||
| class _PyreqwestStream(SyncByteStream): | ||
| def __init__(self, response: PyreqwestSyncResponse, exit_stack: ExitStack) -> None: | ||
| self._reader = response.body_reader | ||
| self._exit_stack = exit_stack | ||
|
|
||
| def __iter__(self) -> Iterator[bytes]: | ||
| while (chunk := self._reader.read()) is not None: | ||
| yield bytes(chunk) | ||
|
|
||
| def close(self) -> None: | ||
| self._exit_stack.close() | ||
|
|
||
|
|
||
| class _AsyncPyreqwestStream(AsyncByteStream): | ||
| def __init__(self, response: PyreqwestResponse, exit_stack: AsyncExitStack) -> None: | ||
| self._reader = response.body_reader | ||
| self._exit_stack = exit_stack | ||
|
|
||
| async def __aiter__(self) -> AsyncIterator[bytes]: | ||
| while (chunk := await self._reader.read()) is not None: | ||
| yield bytes(chunk) | ||
|
|
||
| async def aclose(self) -> None: | ||
| await self._exit_stack.aclose() | ||
|
|
||
|
|
||
| class PyreqwestTransport(BaseTransport): | ||
| """A sync transport backed by pyreqwest/reqwest.""" | ||
|
|
||
| def __init__(self, client: PyreqwestSyncClient | None = None, *, close_client: bool = True) -> None: | ||
| SyncClientBuilder = _load_pyreqwest_sync_client_builder() | ||
| self._client = client or SyncClientBuilder().follow_redirects(False).default_cookie_store(False).build() | ||
| self._close_client = client is None or close_client | ||
|
|
||
| def handle_request(self, request: Request) -> Response: | ||
| content = request.read() | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Streaming uploads are fully buffered before the connection opens, so large or unbounded request iterators lose streaming/backpressure and can exhaust memory. Pass a streaming request body to pyreqwest rather than calling Prompt for AI agents |
||
| builder = self._client.request(request.method, str(request.url)).headers(request.headers.multi_items()) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When a caller supplies a client- or request-level timeout, Useful? React with 👍 / 👎.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Configured Prompt for AI agents |
||
| if content: | ||
| builder = builder.body_bytes(content) | ||
|
|
||
| exit_stack = ExitStack() | ||
| try: | ||
| response = exit_stack.enter_context(builder.build_streamed()) | ||
| return Response( | ||
| response.status, | ||
| headers=list(response.headers.items()), | ||
| stream=_PyreqwestStream(response, exit_stack), | ||
| extensions={"http_version": _http_version(response.version)}, | ||
| ) | ||
| except BaseException: | ||
| exit_stack.close() | ||
| raise | ||
|
Comment on lines
+91
to
+93
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When connection establishment or sending fails inside Useful? React with 👍 / 👎.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Network failures from pyreqwest are re-raised as raw exceptions rather than being mapped to Consider wrapping the except SomePyreqwestConnectionError as exc:
raise httpx2.ConnectError(str(exc)) from excPrompt for AI agents |
||
|
|
||
| def close(self) -> None: | ||
| if self._close_client: | ||
| self._client.close() | ||
|
|
||
|
|
||
| class AsyncPyreqwestTransport(AsyncBaseTransport): | ||
| """An async transport backed by pyreqwest/reqwest.""" | ||
|
|
||
| def __init__(self, client: PyreqwestClient | None = None, *, close_client: bool = True) -> None: | ||
| ClientBuilder = _load_pyreqwest_async_client_builder() | ||
| self._client = client or ClientBuilder().follow_redirects(False).default_cookie_store(False).build() | ||
| self._close_client = client is None or close_client | ||
|
|
||
| async def handle_async_request(self, request: Request) -> Response: | ||
| content = await request.aread() | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Async streaming uploads are fully buffered before the connection opens, so large or unbounded async iterators lose streaming/backpressure and can exhaust memory. Pass a streaming request body to pyreqwest rather than calling Prompt for AI agents |
||
| builder = self._client.request(request.method, str(request.url)).headers(request.headers.multi_items()) | ||
| if content: | ||
| builder = builder.body_bytes(content) | ||
|
|
||
| exit_stack = AsyncExitStack() | ||
| try: | ||
| response = await exit_stack.enter_async_context(builder.build_streamed()) | ||
| return Response( | ||
| response.status, | ||
| headers=list(response.headers.items()), | ||
| stream=_AsyncPyreqwestStream(response, exit_stack), | ||
| extensions={"http_version": _http_version(response.version)}, | ||
| ) | ||
| except BaseException: | ||
| await exit_stack.aclose() | ||
| raise | ||
|
|
||
| async def aclose(self) -> None: | ||
| if self._close_client: | ||
| await self._client.close() | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| from __future__ import annotations | ||
|
|
||
| import asyncio | ||
|
|
||
| import pytest | ||
|
|
||
| import httpx2 | ||
|
|
||
| pytest.importorskip("pyreqwest") | ||
|
|
||
|
|
||
| def test_pyreqwest_transport(server: object) -> None: | ||
| with httpx2.Client(transport=httpx2.PyreqwestTransport()) as client: | ||
| response = client.get(str(server.url)) # type: ignore[attr-defined] | ||
|
|
||
| assert response.status_code == 200 | ||
| assert response.text == "Hello, world!" | ||
| assert response.http_version == "HTTP/1.1" | ||
|
|
||
|
|
||
| def test_async_pyreqwest_transport(server: object) -> None: | ||
| async def run() -> httpx2.Response: | ||
| async with httpx2.AsyncClient(transport=httpx2.AsyncPyreqwestTransport()) as client: | ||
| return await client.get(str(server.url)) # type: ignore[attr-defined] | ||
|
|
||
| response = asyncio.run(run()) | ||
|
|
||
| assert response.status_code == 200 | ||
| assert response.text == "Hello, world!" | ||
| assert response.http_version == "HTTP/1.1" | ||
|
|
||
|
|
||
| def test_pyreqwest_transport_request_body(server: object) -> None: | ||
| with httpx2.Client(transport=httpx2.PyreqwestTransport()) as client: | ||
| response = client.post(str(server.url.copy_with(path="/echo_body")), content=b"hello") # type: ignore[attr-defined] | ||
|
|
||
| assert response.status_code == 200 | ||
| assert response.content == b"hello" |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Those can't be here.