Merge pull request #288 from autoscrape-labs/refactor/type-checking

refactor: refactor type hints for better clarity and future compatibility

Author: thalissonvs

pydoll/browser/chromium/base.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import json
 import logging
@@ -8,11 +10,9 @@
 from contextlib import suppress
 from functools import partial
 from random import randint
-from tempfile import TemporaryDirectory
-from typing import Any, Awaitable, Callable, Optional, overload
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional, overload
 from urllib.parse import urlsplit, urlunsplit
 
-from pydoll.browser.interfaces import BrowserOptionsManager
 from pydoll.browser.managers import (
     BrowserProcessManager,
     ProxyManager,
@@ -35,30 +35,38 @@
     MissingTargetOrWebSocket,
     NoValidTabFound,
 )
-from pydoll.protocol.base import Command, Response, T_CommandParams, T_CommandResponse
-from pydoll.protocol.browser.methods import (
-    GetVersionResponse,
-    GetVersionResult,
-    GetWindowForTargetResponse,
-)
-from pydoll.protocol.browser.types import Bounds, DownloadBehavior, PermissionType
-from pydoll.protocol.fetch.events import FetchEvent, RequestPausedEvent
-from pydoll.protocol.fetch.types import AuthChallengeResponseType, HeaderEntry
-from pydoll.protocol.network.types import (
-    Cookie,
-    CookieParam,
-    ErrorReason,
-    RequestMethod,
-    ResourceType,
-)
-from pydoll.protocol.storage.methods import GetCookiesResponse
-from pydoll.protocol.target.methods import (
-    CreateBrowserContextResponse,
-    CreateTargetResponse,
-    GetBrowserContextsResponse,
-    GetTargetsResponse,
-)
-from pydoll.protocol.target.types import TargetInfo
+from pydoll.protocol.browser.types import DownloadBehavior
+from pydoll.protocol.fetch.events import FetchEvent
+from pydoll.protocol.fetch.types import AuthChallengeResponseType
+
+if TYPE_CHECKING:
+    from tempfile import TemporaryDirectory
+
+    from pydoll.browser.interfaces import BrowserOptionsManager
+    from pydoll.protocol.base import Command, Response, T_CommandParams, T_CommandResponse
+    from pydoll.protocol.browser.methods import (
+        GetVersionResponse,
+        GetVersionResult,
+        GetWindowForTargetResponse,
+    )
+    from pydoll.protocol.browser.types import Bounds, PermissionType
+    from pydoll.protocol.fetch.events import RequestPausedEvent
+    from pydoll.protocol.fetch.types import HeaderEntry
+    from pydoll.protocol.network.types import (
+        Cookie,
+        CookieParam,
+        ErrorReason,
+        RequestMethod,
+        ResourceType,
+    )
+    from pydoll.protocol.storage.methods import GetCookiesResponse
+    from pydoll.protocol.target.methods import (
+        CreateBrowserContextResponse,
+        CreateTargetResponse,
+        GetBrowserContextsResponse,
+        GetTargetsResponse,
+    )
+    from pydoll.protocol.target.types import TargetInfo
 
 logger = logging.getLogger(__name__)
 

pydoll/browser/chromium/chrome.py

@@ -1,13 +1,17 @@
+from __future__ import annotations
+
 import logging
 import platform
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
 from pydoll.browser.chromium.base import Browser
 from pydoll.browser.managers import ChromiumOptionsManager
-from pydoll.browser.options import ChromiumOptions
 from pydoll.exceptions import UnsupportedOS
 from pydoll.utils import validate_browser_paths
 
+if TYPE_CHECKING:
+    from pydoll.browser.options import ChromiumOptions
+
 logger = logging.getLogger(__name__)
 
 

pydoll/browser/chromium/edge.py

@@ -1,13 +1,17 @@
+from __future__ import annotations
+
 import logging
 import platform
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
 from pydoll.browser.chromium.base import Browser
 from pydoll.browser.managers import ChromiumOptionsManager
-from pydoll.browser.options import Options
 from pydoll.exceptions import UnsupportedOS
 from pydoll.utils import validate_browser_paths
 
+if TYPE_CHECKING:
+    from pydoll.browser.options import Options
+
 logger = logging.getLogger(__name__)
 
 

pydoll/browser/managers/browser_options_manager.py

@@ -1,10 +1,15 @@
+from __future__ import annotations
+
 import logging
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
-from pydoll.browser.interfaces import BrowserOptionsManager, Options
+from pydoll.browser.interfaces import BrowserOptionsManager
 from pydoll.browser.options import ChromiumOptions
 from pydoll.exceptions import InvalidOptionsObject
 
+if TYPE_CHECKING:
+    from pydoll.browser.options import Options
+
 logger = logging.getLogger(__name__)
 
 

pydoll/browser/managers/proxy_manager.py

@@ -1,7 +1,10 @@
+from __future__ import annotations
+
 import logging
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
-from pydoll.browser.options import Options
+if TYPE_CHECKING:
+    from pydoll.browser.options import Options
 
 logger = logging.getLogger(__name__)
 

pydoll/browser/requests/request.py

@@ -3,16 +3,19 @@
 It allows making HTTP requests using the browser's fetch API.
 """
 
+from __future__ import annotations
+
 import json as jsonlib
 import logging
 from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 
-from pydoll.browser.requests.response import HTTPError, Response
+from pydoll.browser.requests.response import Response
 from pydoll.commands.runtime_commands import RuntimeCommands
 from pydoll.constants import Scripts
+from pydoll.exceptions import HTTPError
 from pydoll.protocol.fetch.types import HeaderEntry
-from pydoll.protocol.network.events import NetworkEvent
+from pydoll.protocol.network.events import NetworkEvent, ResponseReceivedExtraInfoEventParams
 from pydoll.protocol.network.types import CookieParam
 
 logger = logging.getLogger(__name__)
@@ -27,7 +30,6 @@
         ResponseReceivedEvent,
         ResponseReceivedEventParams,
         ResponseReceivedExtraInfoEvent,
-        ResponseReceivedExtraInfoEventParams,
     )
     from pydoll.protocol.runtime.methods import EvaluateResponse
 
@@ -71,7 +73,7 @@ class Request:
     - Cookies are managed automatically by the browser
     """
 
-    def __init__(self, tab: 'Tab'):
+    def __init__(self, tab: Tab):
         """Initialize a new Request instance bound to a browser tab.
 
         Args:
@@ -81,8 +83,8 @@ def __init__(self, tab: 'Tab'):
         """
         self.tab = tab
         self._network_events_enabled = False
-        self._requests_sent: list['RequestSentEvent'] = []
-        self._requests_received: list['RequestReceivedEvent'] = []
+        self._requests_sent: list[RequestSentEvent] = []
+        self._requests_received: list[RequestReceivedEvent] = []
         logger.debug('Request helper initialized for tab')
 
     async def request(
@@ -334,7 +336,7 @@ def _handle_data_options(
             options['body'] = data
             logger.debug('Request data set as raw payload')
 
-    async def _execute_fetch_request(self, url: str, options: dict[str, Any]) -> 'EvaluateResponse':
+    async def _execute_fetch_request(self, url: str, options: dict[str, Any]) -> EvaluateResponse:
         """Execute the fetch request using browser's runtime."""
         script = Scripts.MAKE_REQUEST.format(url=jsonlib.dumps(url), options=jsonlib.dumps(options))
         await self._register_callbacks()
@@ -350,7 +352,7 @@ async def _execute_fetch_request(self, url: str, options: dict[str, Any]) -> 'Ev
 
     @staticmethod
     def _build_response(
-        result: 'EvaluateResponse',
+        result: EvaluateResponse,
         response_headers: list[HeaderEntry],
         request_headers: list[HeaderEntry],
         cookies: list[CookieParam],
@@ -385,11 +387,11 @@ async def _register_callbacks(self) -> None:
             logger.debug('Network events enabled on tab for request capture')
 
         def append_received_request(event: dict) -> None:
-            self._requests_received.append(cast('RequestReceivedEvent', event))
+            self._requests_received.append(cast(RequestReceivedEvent, event))
             logger.debug(f'Appended received request: event={event}')
 
         def append_sent_request(event: dict) -> None:
-            self._requests_sent.append(cast('RequestSentEvent', event))
+            self._requests_sent.append(cast(RequestSentEvent, event))
             logger.debug(f'Appended sent request: event={event}')
 
         await self.tab.on(
@@ -450,7 +452,7 @@ def _extract_sent_headers(self) -> list[HeaderEntry]:
 
     @staticmethod
     def _extract_headers_from_events(
-        events: Union[list['RequestSentEvent'], list['RequestReceivedEvent']],
+        events: Union[list[RequestSentEvent], list[RequestReceivedEvent]],
         event_extractors: dict[str, Callable[[Any], list[HeaderEntry]]],
     ) -> list[HeaderEntry]:
         """Extract headers from network events using appropriate extractors.
@@ -488,7 +490,7 @@ def _extract_headers_from_events(
         return headers
 
     def _extract_request_sent_headers(
-        self, params: 'RequestWillBeSentEventParams'
+        self, params: RequestWillBeSentEventParams
     ) -> list[HeaderEntry]:
         """Extract headers from main request event.
 
@@ -503,7 +505,7 @@ def _extract_request_sent_headers(
         return self._convert_dict_to_header_entries(request.get('headers', {}))
 
     def _extract_request_sent_extra_info_headers(
-        self, params: 'RequestWillBeSentExtraInfoEventParams'
+        self, params: RequestWillBeSentExtraInfoEventParams
     ) -> list[HeaderEntry]:
         """Extract headers from extra request info event.
 
@@ -520,7 +522,7 @@ def _extract_request_sent_extra_info_headers(
         return self._convert_dict_to_header_entries(params.get('headers', {}))
 
     def _extract_response_received_headers(
-        self, params: 'ResponseReceivedEventParams'
+        self, params: ResponseReceivedEventParams
     ) -> list[HeaderEntry]:
         """Extract headers from main response event.
 
@@ -535,7 +537,7 @@ def _extract_response_received_headers(
         return self._convert_dict_to_header_entries(response.get('headers', {}))
 
     def _extract_response_received_extra_info_headers(
-        self, params: 'ResponseReceivedExtraInfoEventParams'
+        self, params: ResponseReceivedExtraInfoEventParams
     ) -> list[HeaderEntry]:
         """Extract headers from extra response info event.
 
@@ -583,7 +585,7 @@ def _extract_set_cookies(self) -> list[CookieParam]:
             f'response_extra_info_events={response_extra_info_events}'
         )
         for event in response_extra_info_events:
-            params = cast('ResponseReceivedExtraInfoEventParams', event['params'])
+            params = cast(ResponseReceivedExtraInfoEventParams, event['params'])
             headers = self._convert_dict_to_header_entries(params['headers'])
             logger.debug(f'Converting dictionary to header entries: headers={headers}')
             set_cookie_headers = [
@@ -598,7 +600,7 @@ def _extract_set_cookies(self) -> list[CookieParam]:
         logger.debug(f'Set cookies extracted: cookies={cookies}')
         return cookies
 
-    def _filter_response_extra_info_events(self) -> list['RequestReceivedEvent']:
+    def _filter_response_extra_info_events(self) -> list[RequestReceivedEvent]:
         """Filter network events to find those containing Set-Cookie information.
 
         Returns:

pydoll/browser/requests/response.py

@@ -1,9 +1,14 @@
+from __future__ import annotations
+
 import json as jsonlib
 import logging
-from typing import Any, Optional, Union
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+from pydoll.exceptions import HTTPError
 
-from pydoll.protocol.fetch.types import HeaderEntry
-from pydoll.protocol.network.types import CookieParam
+if TYPE_CHECKING:
+    from pydoll.protocol.fetch.types import HeaderEntry
+    from pydoll.protocol.network.types import CookieParam
 
 logger = logging.getLogger(__name__)
 
@@ -221,11 +226,3 @@ def raise_for_status(self) -> None:
                 f'HTTP error status encountered: status={self.status_code}, url={self._url}'
             )
             raise HTTPError(f'{self.status_code} Client Error: for url {self._url}')
-
-
-class HTTPError(Exception):
-    """
-    Exception raised for HTTP error responses (4xx and 5xx status codes).
-    """
-
-    pass