Adapt all tests for Windows

Author: elacuesta

scrapy_playwright/_utils.py

@@ -117,6 +117,7 @@ def get_event_loop(cls) -> asyncio.AbstractEventLoop:
                 if not cls.loop.is_running():
                     cls.thread = threading.Thread(target=cls.loop.run_forever, daemon=True)
                     cls.thread.start()
+                    logger.info("Started loop on separate thread: %s", cls.loop)
             return cls.loop
 
         @classmethod

tests/tests_asyncio/test_browser_contexts.py

@@ -10,11 +10,12 @@
 from scrapy import Spider, Request
 from scrapy_playwright.page import PageMethod
 
-from tests import make_handler
+from tests import allow_windows, make_handler
 from tests.mockserver import StaticMockServer
 
 
 class MixinTestCaseMultipleContexts:
+    @allow_windows
     async def test_context_kwargs(self):
         settings_dict = {
             "PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
@@ -37,6 +38,7 @@ async def test_context_kwargs(self):
                 with pytest.raises(PlaywrightTimeoutError):
                     await handler._download_request(req, Spider("foo"))
 
+    @allow_windows
     async def test_contexts_max_pages(self):
         settings = {
             "PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
@@ -71,6 +73,7 @@ async def test_contexts_max_pages(self):
 
             assert handler.stats.get_value("playwright/page_count/max_concurrent") == 4
 
+    @allow_windows
     async def test_max_contexts(self):
         def cb_close_context(task):
             response = task.result()
@@ -105,6 +108,7 @@ def cb_close_context(task):
 
             assert handler.stats.get_value("playwright/context_count/max_concurrent") == 4
 
+    @allow_windows
     async def test_contexts_startup(self):
         settings = {
             "PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
@@ -143,6 +147,7 @@ async def test_contexts_startup(self):
             assert cookie["value"] == "bar"
             assert cookie["domain"] == "example.org"
 
+    @allow_windows
     async def test_persistent_context(self):
         temp_dir = f"{tempfile.gettempdir()}/{uuid4()}"
         settings = {
@@ -161,6 +166,7 @@ async def test_persistent_context(self):
             assert handler.context_wrappers["persistent"].persistent
             assert not hasattr(handler, "browser")
 
+    @allow_windows
     async def test_mixed_persistent_contexts(self):
         temp_dir = f"{tempfile.gettempdir()}/{uuid4()}"
         settings = {
@@ -183,6 +189,7 @@ async def test_mixed_persistent_contexts(self):
             assert not handler.context_wrappers["non-persistent"].persistent
             assert isinstance(handler.browser, Browser)
 
+    @allow_windows
     async def test_contexts_dynamic(self):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
             assert len(handler.context_wrappers) == 0

tests/tests_asyncio/test_extensions.py

@@ -1,3 +1,4 @@
+import platform
 from asyncio.subprocess import Process as AsyncioProcess
 from unittest import IsolatedAsyncioTestCase
 from unittest.mock import MagicMock, patch
@@ -34,6 +35,10 @@ class MockMemoryInfo:
     rss = 999
 
 
+@pytest.mark.skipif(
+    platform.system() == "Windows",
+    reason="resource stdlib module is not available on Windows",
+)
 @patch("scrapy.extensions.memusage.MailSender")
 class TestMemoryUsageExtension(IsolatedAsyncioTestCase):
     async def test_process_availability(self, _MailSender):

tests/tests_asyncio/test_headers.py

@@ -5,11 +5,12 @@
 import pytest
 from scrapy import Spider, Request
 
-from tests import make_handler
+from tests import allow_windows, make_handler
 from tests.mockserver import MockServer
 
 
 class MixinProcessHeadersTestCase:
+    @allow_windows
     async def test_user_agent(self):
         settings_dict = {
             "PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
@@ -39,6 +40,7 @@ async def test_user_agent(self):
                 headers = {key.lower(): value for key, value in headers.items()}
                 assert headers["user-agent"] == "foobar"
 
+    @allow_windows
     async def test_playwright_headers(self):
         """Ignore Scrapy headers"""
         settings_dict = {
@@ -63,6 +65,7 @@ async def test_playwright_headers(self):
                 assert "asdf" not in req.headers
                 assert b"asdf" not in req.headers
 
+    @allow_windows
     async def test_use_custom_headers(self):
         """Custom header processing function"""
 

tests/tests_asyncio/test_page_methods.py

@@ -10,7 +10,7 @@
 
 from scrapy_playwright.page import PageMethod
 
-from tests import make_handler, assert_correct_response
+from tests import allow_windows, make_handler, assert_correct_response
 from tests.mockserver import StaticMockServer
 
 
@@ -24,6 +24,7 @@ def get_mimetype(file):
 
 
 class TestPageMethods(IsolatedAsyncioTestCase):
+    @allow_windows
     async def test_page_methods(self):
         screenshot = PageMethod("screenshot", "foo", 123, path="/tmp/file", type="png")
         assert screenshot.method == "screenshot"
@@ -39,6 +40,7 @@ def inject_fixtures(self, caplog):
         caplog.set_level(logging.DEBUG)
         self._caplog = caplog
 
+    @allow_windows
     async def test_page_non_page_method(self):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
             with StaticMockServer() as server:
@@ -63,6 +65,7 @@ async def test_page_non_page_method(self):
                 f"Ignoring {repr(obj)}: expected PageMethod, got {repr(type(obj))}",
             ) in self._caplog.record_tuples
 
+    @allow_windows
     async def test_page_mixed_page_methods(self):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
             with StaticMockServer() as server:
@@ -89,6 +92,7 @@ async def test_page_mixed_page_methods(self):
         assert not req.meta["playwright_page_methods"]["is_closed"].result
         assert req.meta["playwright_page_methods"]["title"].result == "Awesome site"
 
+    @allow_windows
     async def test_page_method_navigation(self):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
             with StaticMockServer() as server:
@@ -110,6 +114,7 @@ async def test_page_method_navigation(self):
             text = resp.css("p::text").get()
             assert text == "Lorem ipsum dolor sit amet, consectetur adipiscing elit."
 
+    @allow_windows
     async def test_page_method_infinite_scroll(self):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
             with StaticMockServer() as server:
@@ -136,6 +141,7 @@ async def test_page_method_infinite_scroll(self):
             assert_correct_response(resp, req)
             assert len(resp.css("div.quote")) == 30
 
+    @allow_windows
     async def test_page_method_screenshot(self):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
             with NamedTemporaryFile(mode="w+b", delete=False) as png_file:
@@ -156,6 +162,7 @@ async def test_page_method_screenshot(self):
                 if platform.system() != "Windows":
                     assert get_mimetype(png_file) == "image/png"
 
+    @allow_windows
     async def test_page_method_pdf(self):
         if self.browser_type != "chromium":
             pytest.skip("PDF generation is supported only in Chromium")

tests/tests_asyncio/test_playwright_requests.py

@@ -55,6 +55,7 @@ async def test_basic_response(self):
             assert resp.meta["playwright_page"].url == resp.url
             await resp.meta["playwright_page"].close()
 
+    @allow_windows
     async def test_post_request(self):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
             with MockServer() as server:
@@ -66,6 +67,7 @@ async def test_post_request(self):
             assert_correct_response(resp, req)
             assert "Request body: foo=bar" in resp.text
 
+    @allow_windows
     async def test_timeout_error(self):
         settings_dict = {
             "PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
@@ -83,6 +85,7 @@ async def test_timeout_error(self):
                     f" exc_type={type(excinfo.value)} exc_msg={str(excinfo.value)}",
                 ) in self._caplog.record_tuples
 
+    @allow_windows
     async def test_retry_page_content_still_navigating(self):
         if self.browser_type != "chromium":
             pytest.skip("Only Chromium seems to redirect meta tags within the same goto call")
@@ -104,6 +107,7 @@ async def test_retry_page_content_still_navigating(self):
             ) in self._caplog.record_tuples
 
     @patch("scrapy_playwright.handler.logger")
+    @allow_windows
     async def test_route_continue_exception(self, logger):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
             scrapy_request = Request(url="https://example.org", method="GET")
@@ -154,6 +158,7 @@ async def test_route_continue_exception(self, logger):
             with pytest.raises(PlaywrightError):
                 await req_handler(route, playwright_request)
 
+    @allow_windows
     async def test_event_handler_dialog_callable(self):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
             with StaticMockServer() as server:
@@ -175,6 +180,7 @@ async def test_event_handler_dialog_callable(self):
 
             assert spider.dialog_message == "foobar"
 
+    @allow_windows
     async def test_event_handler_dialog_str(self):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
             with StaticMockServer() as server:
@@ -196,6 +202,7 @@ async def test_event_handler_dialog_str(self):
 
             assert spider.dialog_message == "foobar"
 
+    @allow_windows
     async def test_event_handler_dialog_missing(self):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
             with StaticMockServer() as server:
@@ -219,6 +226,7 @@ async def test_event_handler_dialog_missing(self):
         ) in self._caplog.record_tuples
         assert getattr(spider, "dialog_message", None) is None
 
+    @allow_windows
     async def test_response_attributes(self):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
             with MockServer() as server:
@@ -230,6 +238,7 @@ async def test_response_attributes(self):
 
         assert response.ip_address == ip_address(server.address)
 
+    @allow_windows
     async def test_page_goto_kwargs_referer(self):
         if self.browser_type != "chromium":
             pytest.skip("referer as goto kwarg seems to work only with chromium :shrug:")
@@ -248,6 +257,7 @@ async def test_page_goto_kwargs_referer(self):
         headers = json.loads(response.css("pre::text").get())
         assert headers["Referer"] == fake_referer
 
+    @allow_windows
     async def test_navigation_returns_none(self):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
             with MockServer():
@@ -263,6 +273,7 @@ async def test_navigation_returns_none(self):
         assert not response.headers
         assert response.status == 200
 
+    @allow_windows
     async def test_abort_requests(self):
         async def should_abort_request_async(request):
             return request.resource_type == "image"
@@ -295,6 +306,7 @@ def should_abort_request_sync(request):
                     assert handler.stats.get_value(f"{resp_prefix}/resource_type/image") is None
                     assert handler.stats.get_value(f"{req_prefix}/aborted") == 3
 
+    @allow_windows
     async def test_page_initialization_ok(self):
         async def init_page(page, _request):
             await page.set_extra_http_headers({"Extra-Header": "Qwerty"})
@@ -315,6 +327,7 @@ async def init_page(page, _request):
         headers = {key.lower(): value for key, value in headers.items()}
         assert headers["extra-header"] == "Qwerty"
 
+    @allow_windows
     async def test_page_initialization_fail(self):
         async def init_page(page, _request, _missing):
             await page.set_extra_http_headers({"Extra-Header": "Qwerty"})
@@ -341,6 +354,7 @@ async def init_page(page, _request, _missing):
                 assert f"[Context=default] Page init callback exception for {req!r}" in entry[2]
                 assert "init_page() missing 1 required positional argument: '_missing'" in entry[2]
 
+    @allow_windows
     async def test_redirect(self):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
             with MockServer() as server:
@@ -358,6 +372,7 @@ async def test_redirect(self):
             server.urljoin("/redirect"),
         ]
 
+    @allow_windows
     async def test_logging_record_spider(self):
         """Make sure at least one log record has the spider as an attribute
         (records sent before opening the spider will not have it).
@@ -370,6 +385,7 @@ async def test_logging_record_spider(self):
 
         assert any(getattr(rec, "spider", None) is spider for rec in self._caplog.records)
 
+    @allow_windows
     async def test_download_file(self):
         settings_dict = {
             "PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
@@ -385,6 +401,7 @@ async def test_download_file(self):
                 assert response.body.startswith(b"%PDF-1.5")
                 assert handler.stats.get_value("playwright/download_count") == 1
 
+    @allow_windows
     async def test_download_file_delay_ok(self):
         settings_dict = {
             "PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
@@ -401,6 +418,7 @@ async def test_download_file_delay_ok(self):
                 assert response.body.startswith(b"%PDF-1.5")
                 assert handler.stats.get_value("playwright/download_count") == 1
 
+    @allow_windows
     async def test_download_file_delay_error(self):
         settings_dict = {
             "PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
@@ -421,6 +439,7 @@ async def test_download_file_delay_error(self):
                     f" exc_type={type(excinfo.value)} exc_msg={str(excinfo.value)}",
                 ) in self._caplog.record_tuples
 
+    @allow_windows
     async def test_download_file_failure(self):
         if self.browser_type != "chromium":
             pytest.skip()

tests/tests_asyncio/test_remote.py

@@ -10,7 +10,7 @@
 from playwright.async_api import async_playwright
 from scrapy import Request, Spider
 
-from tests import make_handler, assert_correct_response
+from tests import allow_windows, make_handler, assert_correct_response
 from tests.mockserver import StaticMockServer
 
 
@@ -61,6 +61,7 @@ def inject_fixtures(self, caplog):
         caplog.set_level(logging.DEBUG)
         self._caplog = caplog
 
+    @allow_windows
     async def test_devtools(self):
         async with remote_chromium() as devtools_url:
             settings_dict = {

tests/tests_asyncio/test_settings.py

@@ -4,7 +4,7 @@
 
 from scrapy_playwright.handler import Config
 
-from tests import make_handler
+from tests import allow_windows, make_handler
 
 
 class TestSettings(IsolatedAsyncioTestCase):
@@ -31,6 +31,7 @@ async def test_max_pages_per_context(self):
         config = Config.from_settings(Settings({"CONCURRENT_REQUESTS": 9876}))
         assert config.max_pages_per_context == 9876
 
+    @allow_windows
     async def test_max_contexts(self):
         async with make_handler({"PLAYWRIGHT_MAX_CONTEXTS": None}) as handler:
             assert not hasattr(handler, "context_semaphore")

tests/tests_twisted/test_mixed_requests.py

@@ -10,10 +10,7 @@
 from tests.mockserver import StaticMockServer
 
 
-@pytest.mark.skipif(
-    platform.system() == "Windows",
-    reason="On Windows, If the task reaches two, it will be blocked.",
-)
+@pytest.mark.skipif(platform.system() == "Windows", reason="Gets stuck on Windows")
 class MixedRequestsTestCase(TestCase):
     """
     This test case ensures the handler's 'download_request' method works as expected, and