Only register request and response loggers when needed (#336)

scrapy-plugins · Feb 19, 2025 · de11854 · de11854
1 parent a2a844f
commit de11854
Show file tree

Hide file tree

Showing 2 changed files with 55 additions and 2 deletions.
diff --git a/scrapy_playwright/handler.py b/scrapy_playwright/handler.py
@@ -324,10 +324,11 @@ async def _create_page(self, request: Request, spider: Spider) -> Page:
 
         page.on("close", self._make_close_page_callback(context_name))
         page.on("crash", self._make_close_page_callback(context_name))
-        page.on("request", _make_request_logger(context_name, spider))
-        page.on("response", _make_response_logger(context_name, spider))
         page.on("request", self._increment_request_stats)
         page.on("response", self._increment_response_stats)
+        if logger.getEffectiveLevel() <= logging.DEBUG:
+            page.on("request", _make_request_logger(context_name, spider))
+            page.on("response", _make_response_logger(context_name, spider))
 
         return page
 

diff --git a/tests/tests_asyncio/test_playwright_requests.py b/tests/tests_asyncio/test_playwright_requests.py
@@ -389,6 +389,58 @@ async def test_logging_record_spider(self):
 
         assert any(getattr(rec, "spider", None) is spider for rec in self._caplog.records)
 
+    @allow_windows
+    @patch("scrapy_playwright.handler._make_request_logger")
+    async def test_request_logger_disabled(self, make_request_logger: MagicMock):
+        self._caplog.set_level(logging.DEBUG + 1, "scrapy-playwright")
+        async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
+            with MockServer() as server:
+                req = Request(url=server.urljoin("/index.html"), meta={"playwright": True})
+                await handler._download_request(req, Spider("foo"))
+
+        debug_message = (
+            f"[Context=default] Request: <{req.method} {req.url}> (resource type: document)"
+        )
+        assert not any(rec.message == debug_message for rec in self._caplog.records)
+        make_request_logger.assert_not_called()
+
+    @allow_windows
+    async def test_request_logger_enabled(self):
+        self._caplog.set_level(logging.DEBUG, "scrapy-playwright")
+        async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
+            with MockServer() as server:
+                req = Request(url=server.urljoin("/index.html"), meta={"playwright": True})
+                await handler._download_request(req, Spider("foo"))
+
+        debug_message = (
+            f"[Context=default] Request: <{req.method} {req.url}> (resource type: document)"
+        )
+        assert any(rec.message == debug_message for rec in self._caplog.records)
+
+    @allow_windows
+    @patch("scrapy_playwright.handler._make_response_logger")
+    async def test_response_logger_disabled(self, make_response_logger: MagicMock):
+        self._caplog.set_level(logging.DEBUG + 1, "scrapy-playwright")
+        async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
+            with MockServer() as server:
+                req = Request(url=server.urljoin("/index.html"), meta={"playwright": True})
+                response = await handler._download_request(req, Spider("foo"))
+
+        debug_message = f"[Context=default] Response: <{response.status} {response.url}>"
+        assert not any(rec.message == debug_message for rec in self._caplog.records)
+        make_response_logger.assert_not_called()
+
+    @allow_windows
+    async def test_response_logger_enabled(self):
+        self._caplog.set_level(logging.DEBUG, "scrapy-playwright")
+        async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
+            with MockServer() as server:
+                request = Request(url=server.urljoin("/index.html"), meta={"playwright": True})
+                response = await handler._download_request(request, Spider("foo"))
+
+        debug_message = f"[Context=default] Response: <{response.status} {response.url}>"
+        assert any(rec.message == debug_message for rec in self._caplog.records)
+
     @allow_windows
     async def test_download_file_ok(self):
         settings_dict = {