reorganize page_inputs.py as a submodule; move HttpClient to it

Author: BurnzZ

tests/test_requests.py

@@ -3,15 +3,13 @@
 import pytest
 from web_poet.exceptions import RequestBackendError
 from web_poet.page_inputs import (
+    HttpClient,
     HttpRequest,
     HttpResponse,
     HttpRequestBody,
     HttpRequestHeaders
 )
-from web_poet.requests import (
-    HttpClient,
-    request_backend_var,
-)
+from web_poet.requests import request_backend_var
 
 
 @pytest.fixture
@@ -47,7 +45,7 @@ async def test_perform_request_from_httpclient(async_mock):
 async def test_http_client_single_requests(async_mock):
     client = HttpClient(async_mock)
 
-    with mock.patch("web_poet.requests.HttpRequest") as mock_request:
+    with mock.patch("web_poet.page_inputs.client.HttpRequest") as mock_request:
         response = await client.request("url")
         response.url == "url"
 

web_poet/__init__.py

@@ -1,10 +1,8 @@
 from .pages import WebPage, ItemPage, ItemWebPage, Injectable
-from .requests import (
-    request_backend_var,
-    HttpClient,
-)
+from .requests import request_backend_var
 from .page_inputs import (
     Meta,
+    HttpClient,
     HttpRequest,
     HttpResponse,
     HttpRequestHeaders,

web_poet/page_inputs/__init__.py

@@ -0,0 +1,10 @@
+from .meta import Meta
+from .client import HttpClient
+from .http import (
+    HttpRequest,
+    HttpResponse,
+    HttpRequestHeaders,
+    HttpResponseHeaders,
+    HttpRequestBody,
+    HttpResponseBody,
+)

web_poet/page_inputs/client.py

@@ -0,0 +1,153 @@
+"""This module has a full support for :mod:`asyncio` that enables developers to
+perform asynchronous additional requests inside of Page Objects.
+
+Note that the implementation to fully execute any :class:`~.Request` is not
+handled in this module. With that, the framework using **web-poet** must supply
+the implementation.
+
+You can read more about this in the :ref:`advanced-downloader-impl` documentation.
+"""
+
+import asyncio
+import logging
+from typing import Optional, Dict, List, Union, Callable
+
+from web_poet.requests import request_backend_var
+from web_poet.exceptions import RequestBackendError
+from web_poet.page_inputs.http import (
+    HttpRequest,
+    HttpRequestHeaders,
+    HttpRequestBody,
+    HttpResponse,
+)
+
+logger = logging.getLogger(__name__)
+
+_StrMapping = Dict[str, str]
+_Headers = Union[_StrMapping, HttpRequestHeaders]
+_Body = Union[bytes, HttpRequestBody]
+
+
+async def _perform_request(request: HttpRequest) -> HttpResponse:
+    """Given a :class:`~.Request`, execute it using the **request implementation**
+    that was set in the ``web_poet.request_backend_var`` :mod:`contextvars`
+    instance.
+
+    .. warning::
+        By convention, this function should return a :class:`~.HttpResponse`.
+        However, the underlying downloader assigned in
+        ``web_poet.request_backend_var`` might change that, depending on
+        how the framework using **web-poet** implements it.
+    """
+
+    logger.info(f"Requesting page: {request}")
+
+    try:
+        request_backend = request_backend_var.get()
+    except LookupError:
+        raise RequestBackendError(
+            "Additional requests are used inside the Page Object but the "
+            "current framework has not set any HttpRequest Backend via "
+            "'web_poet.request_backend_var'"
+        )
+
+    response_data: HttpResponse = await request_backend(request)
+    return response_data
+
+
+class HttpClient:
+    """A convenient client to easily execute requests.
+
+    By default, it uses the request implementation assigned in the
+    ``web_poet.request_backend_var`` which is a :mod:`contextvars` instance to
+    download the actual requests. However, it can easily be overridable by
+    providing an optional ``request_downloader`` callable.
+
+    Providing the request implementation by dependency injection would be a good
+    alternative solution when you want to avoid setting up :mod:`contextvars`
+    like ``web_poet.request_backend_var``.
+
+    In any case, this doesn't contain any implementation about how to execute
+    any requests fed into it. When setting that up, make sure that the downloader
+    implementation returns a :class:`~.HttpResponse` instance.
+    """
+
+    def __init__(self, request_downloader: Callable = None):
+        self._request_downloader = request_downloader or _perform_request
+
+    async def request(
+        self,
+        url: str,
+        *,
+        method: str = "GET",
+        headers: Optional[_Headers] = None,
+        body: Optional[_Body] = None,
+    ) -> HttpResponse:
+        """This is a shortcut for creating a :class:`~.HttpRequest` instance and executing
+        that request.
+
+        A :class:`~.HttpResponse` instance should then be returned.
+
+        .. warning::
+            By convention, the request implementation supplied optionally to
+            :class:`~.HttpClient` should return a :class:`~.HttpResponse` instance.
+            However, the underlying implementation supplied might change that,
+            depending on how the framework using **web-poet** implements it.
+        """
+        headers = headers or {}
+        body = body or b""
+        req = HttpRequest(url=url, method=method, headers=headers, body=body)
+        return await self.execute(req)
+
+    async def get(
+        self, url: str, *, headers: Optional[_Headers] = None
+    ) -> HttpResponse:
+        """Similar to :meth:`~.HttpClient.request` but peforming a ``GET``
+        request.
+        """
+        return await self.request(url=url, method="GET", headers=headers)
+
+    async def post(
+        self,
+        url: str,
+        *,
+        headers: Optional[_Headers] = None,
+        body: Optional[_Body] = None,
+    ) -> HttpResponse:
+        """Similar to :meth:`~.HttpClient.request` but performing a ``POST``
+        request.
+        """
+        return await self.request(url=url, method="POST", headers=headers, body=body)
+
+    async def execute(self, request: HttpRequest) -> HttpResponse:
+        """Accepts a single instance of :class:`~.HttpRequest` and executes it
+        using the request implementation configured in the :class:`~.HttpClient`
+        instance.
+
+        This returns a single :class:`~.HttpResponse`.
+        """
+        return await self._request_downloader(request)
+
+    async def batch_execute(
+        self, *requests: HttpRequest, return_exceptions: bool = False
+    ) -> List[Union[HttpResponse, Exception]]:
+        """Similar to :meth:`~.HttpClient.execute` but accepts a collection of
+        :class:`~.HttpRequest` instances that would be batch executed.
+
+        The order of the :class:`~.HttpResponses` would correspond to the order
+        of :class:`~.HttpRequest` passed.
+
+        If any of the :class:`~.HttpRequest` raises an exception upon execution,
+        the exception is raised.
+
+        To prevent this, the actual exception can be returned alongside any
+        successful :class:`~.HttpResponse`. This enables salvaging any usable
+        responses despite any possible failures. This can be done by setting
+        ``True`` to the ``return_exceptions`` parameter.
+        """
+
+        coroutines = [self._request_downloader(r) for r in requests]
+        responses = await asyncio.gather(
+            *coroutines, return_exceptions=return_exceptions
+        )
+        return responses

web_poet/page_inputs.py renamed to web_poet/page_inputs/http.py

@@ -14,7 +14,8 @@
 from web_poet.utils import memoizemethod_noargs
 
 T_headers = TypeVar("T_headers", bound="HttpResponseHeaders")
-AnyStrDict = Dict[AnyStr, Union[AnyStr, List[AnyStr], Tuple[AnyStr, ...]]]
+
+_AnyStrDict = Dict[AnyStr, Union[AnyStr, List[AnyStr], Tuple[AnyStr, ...]]]
 
 
 class HttpRequestBody(bytes):
@@ -99,7 +100,7 @@ class HttpResponseHeaders(_HttpHeaders):
 
     @classmethod
     def from_bytes_dict(
-        cls: Type[T_headers], arg: AnyStrDict, encoding: str = "utf-8"
+        cls: Type[T_headers], arg: _AnyStrDict, encoding: str = "utf-8"
     ) -> T_headers:
         """An alternative constructor for instantiation where the header-value
         pairs could be in raw bytes form.
@@ -270,13 +271,3 @@ def _auto_detect_fun(self, body: bytes) -> Optional[str]:
             except UnicodeError:
                 continue
             return resolve_encoding(enc)
-
-
-class Meta(dict):
-    """Container class that could contain any arbitrary data to be passed into
-    a Page Object.
-
-    Note that this is simply a subclass of Python's ``dict``.
-    """
-
-    pass

web_poet/page_inputs/meta.py

@@ -0,0 +1,8 @@
+class Meta(dict):
+    """Container class that could contain any arbitrary data to be passed into
+    a Page Object.
+
+    Note that this is simply a subclass of Python's ``dict``.
+    """
+
+    pass