refactor HttpClient's request interface

Author: BurnzZ

tests/test_requests.py

@@ -3,7 +3,7 @@
 import pytest
 from web_poet.page_inputs import ResponseData
 from web_poet.requests import (
-    GenericRequest,
+    Request,
     perform_request,
     HttpClient,
     RequestBackendError,
@@ -25,13 +25,13 @@ async def async_test(req):
 
 def test_generic_request():
 
-    req = GenericRequest("url")
+    req = Request("url")
     assert req.url == "url"
     assert req.method == "GET"
     assert req.headers is None
     assert req.body is None
 
-    req = GenericRequest(
+    req = Request(
         "url", method="POST", headers={"User-Agent": "test agent"}, body=b"body"
     )
     assert req.method == "POST"
@@ -42,7 +42,7 @@ def test_generic_request():
 @pytest.mark.asyncio
 async def test_perform_request(async_mock):
 
-    req = GenericRequest("url")
+    req = Request("url")
 
     with pytest.raises(RequestBackendError):
         await perform_request(req)
@@ -56,16 +56,36 @@ async def test_perform_request(async_mock):
 
 
 @pytest.mark.asyncio
-async def test_http_client(async_mock):
+async def test_http_client_single_requests(async_mock):
     client = HttpClient(async_mock)
     assert client.request_downloader == async_mock
 
-    req_1 = GenericRequest("url-1")
-    req_2 = GenericRequest("url-2")
+    with mock.patch("web_poet.requests.Request") as mock_request:
+        response = await client.request("url")
+        response.url == "url"
 
-    # It should be able to accept arbitrary number of requests
-    client.request(req_1)
-    responses = await client.request(req_1, req_2)
+        response = await client.get("url-get")
+        response.url == "url-get"
 
-    assert responses[0].url == req_1.url
-    assert responses[1].url == req_2.url
+        response = await client.post("url-post")
+        response.url == "url-post"
+
+        assert mock_request.call_args_list == [
+            mock.call("url", "GET", None, None),
+            mock.call("url-get", "GET", None, None),
+            mock.call("url-post", "POST", None, None),
+        ]
+
+
+@pytest.mark.asyncio
+async def test_http_client_batch_requests(async_mock):
+    client = HttpClient(async_mock)
+
+    requests = [
+        Request("url-1"),
+        Request("url-get", method="GET"),
+        Request("url-post", method="POST"),
+    ]
+    responses = await client.batch_requests(*requests)
+
+    assert all([isinstance(response, ResponseData) for response in responses])

web_poet/__init__.py

@@ -1,3 +1,3 @@
 from .pages import WebPage, ItemPage, ItemWebPage, Injectable
-from .page_inputs import ResponseData
-from .requests import request_backend_var, GenericRequest, HttpClient
+from .page_inputs import ResponseData, Meta
+from .requests import request_backend_var, Request, HttpClient

web_poet/requests.py

@@ -50,9 +50,13 @@
 
 import attr
 
+from web_poet.page_inputs import ResponseData
+
 logger = logging.getLogger(__name__)
 
 
+mapping = Dict[Union[str, ByteString], Any]
+
 # Frameworks that wants to support additional requests in ``web-poet`` should
 # set the appropriate implementation for requesting data.
 request_backend_var: ContextVar = ContextVar("request_backend")
@@ -63,16 +67,16 @@ class RequestBackendError(Exception):
 
 
 @attr.define
-class GenericRequest:
+class Request:
     """Represents a generic HTTP request."""
 
     url: str
     method: str = "GET"
-    headers: Optional[Dict[Union[str, ByteString], Any]] = None
+    headers: Optional[mapping] = None
     body: Optional[str] = None
 
 
-async def perform_request(request: GenericRequest):
+async def perform_request(request: Request):
     logger.info(f"Requesting page: {request}")
 
     try:
@@ -92,7 +96,25 @@ class HttpClient:
     def __init__(self, request_downloader=None):
         self.request_downloader = request_downloader or perform_request
 
-    async def request(self, *requests: List[GenericRequest]):
+    async def request(
+        self,
+        url: str,
+        method: str = "GET",
+        headers: Optional[mapping] = None,
+        body: Optional[str] = None,
+    ) -> ResponseData:
+        r = Request(url, method, headers, body)
+        return await self.request_downloader(r)
+
+    async def get(self, url: str, headers: Optional[mapping] = None) -> ResponseData:
+        return await self.request(url=url, method="GET", headers=headers)
+
+    async def post(
+        self, url: str, headers: Optional[mapping] = None, body: Optional[str] = None
+    ) -> ResponseData:
+        return await self.request(url=url, method="POST", headers=headers, body=body)
+
+    async def batch_requests(self, *requests: List[Request]):
         coroutines = [self.request_downloader(r) for r in requests]
         responses = await asyncio.gather(*coroutines)
         return responses