feature: async support (#225)

phanak-sap · mnunzio · Albo90 · web-flow · commit fd28f5f22063 · 2022-09-05T19:56:35.000+02:00
Add async networking libraries support

Co-authored-by: mnunzio &lt;mnunzio90@gmail.com&gt;
Co-authored-by: Alberto Moio &lt;moioalberto@gmail.com&gt;
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -14,7 +14,7 @@ jobs:
 
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10.1"]
         lxml-version: ["4.1.1", "4.2.6", "4.3.5", "4.4.3", "4.5.2", "4.6.5", "4.7.1", "4.8.0", "4.9.1"]
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -1,6 +1,9 @@
 requests>=2.28.1
 pytest>=7.1.2
+pytest-aiohttp>=1.0.4
 responses>=0.21.0
+httpx>=0.23.0
+respx>=0.19.2
 setuptools>=38.2.4
 setuptools-scm>=1.15.6
 pylint==2.8.3
diff --git a/pyodata/client.py b/pyodata/client.py
@@ -8,11 +8,27 @@
 from pyodata.exceptions import PyODataException, HttpError
 
 
+async def _async_fetch_metadata(connection, url, logger):
+    logger.info('Fetching metadata')
+
+    async with connection.get(url + '$metadata') as async_response:
+        resp = pyodata.v2.service.ODataHttpResponse(url=async_response.url,
+                                                    headers=async_response.headers,
+                                                    status_code=async_response.status,
+                                                    content=await async_response.read())
+
+        return _common_fetch_metadata(resp, logger)
+
+
 def _fetch_metadata(connection, url, logger):
     # download metadata
     logger.info('Fetching metadata')
     resp = connection.get(url + '$metadata')
 
+    return _common_fetch_metadata(resp, logger)
+
+
+def _common_fetch_metadata(resp, logger):
     logger.debug('Retrieved the response:\n%s\n%s',
                  '\n'.join((f'H: {key}: {value}' for key, value in resp.headers.items())),
                  resp.content)
@@ -37,6 +53,25 @@ class Client:
 
     ODATA_VERSION_2 = 2
 
+    @staticmethod
+    async def build_async_client(url, connection, odata_version=ODATA_VERSION_2, namespaces=None,
+                                 config: pyodata.v2.model.Config = None, metadata: str = None):
+        """Create instance of the OData Client for given URL"""
+
+        logger = logging.getLogger('pyodata.client')
+
+        if odata_version == Client.ODATA_VERSION_2:
+
+            # sanitize url
+            url = url.rstrip('/') + '/'
+
+            if metadata is None:
+                metadata = await _async_fetch_metadata(connection, url, logger)
+            else:
+                logger.info('Using static metadata')
+            return Client._build_service(logger, url, connection, odata_version, namespaces, config, metadata)
+        raise PyODataException(f'No implementation for selected odata version {odata_version}')
+
     def __new__(cls, url, connection, odata_version=ODATA_VERSION_2, namespaces=None,
                 config: pyodata.v2.model.Config = None, metadata: str = None):
         """Create instance of the OData Client for given URL"""
@@ -53,24 +88,29 @@ def __new__(cls, url, connection, odata_version=ODATA_VERSION_2, namespaces=None
             else:
                 logger.info('Using static metadata')
 
-            if config is not None and namespaces is not None:
-                raise PyODataException('You cannot pass namespaces and config at the same time')
+            return Client._build_service(logger, url, connection, odata_version, namespaces, config, metadata)
+        raise PyODataException(f'No implementation for selected odata version {odata_version}')
 
-            if config is None:
-                config = pyodata.v2.model.Config()
+    @staticmethod
+    def _build_service(logger, url, connection, odata_version=ODATA_VERSION_2, namespaces=None,
+                       config: pyodata.v2.model.Config = None, metadata: str = None):
 
-            if namespaces is not None:
-                warnings.warn("Passing namespaces directly is deprecated. Use class Config instead", DeprecationWarning)
-                config.namespaces = namespaces
+        if config is not None and namespaces is not None:
+            raise PyODataException('You cannot pass namespaces and config at the same time')
 
-            # create model instance from received metadata
-            logger.info('Creating OData Schema (version: %d)', odata_version)
-            schema = pyodata.v2.model.MetadataBuilder(metadata, config=config).build()
+        if config is None:
+            config = pyodata.v2.model.Config()
 
-            # create service instance based on model we have
-            logger.info('Creating OData Service (version: %d)', odata_version)
-            service = pyodata.v2.service.Service(url, schema, connection, config=config)
+        if namespaces is not None:
+            warnings.warn("Passing namespaces directly is deprecated. Use class Config instead", DeprecationWarning)
+            config.namespaces = namespaces
 
-            return service
+        # create model instance from received metadata
+        logger.info('Creating OData Schema (version: %d)', odata_version)
+        schema = pyodata.v2.model.MetadataBuilder(metadata, config=config).build()
 
-        raise PyODataException(f'No implementation for selected odata version {odata_version}')
+        # create service instance based on model we have
+        logger.info('Creating OData Service (version: %d)', odata_version)
+        service = pyodata.v2.service.Service(url, schema, connection, config=config)
+
+        return service
diff --git a/pyodata/v2/service.py b/pyodata/v2/service.py
@@ -102,7 +102,8 @@ def decode(message):
 class ODataHttpResponse:
     """Representation of http response"""
 
-    def __init__(self, headers, status_code, content=None):
+    def __init__(self, headers, status_code, content=None, url=None):
+        self.url = url
         self.headers = headers
         self.status_code = status_code
         self.content = content
@@ -292,14 +293,7 @@ def add_headers(self, value):
 
         self._headers.update(value)
 
-    def execute(self):
-        """Fetches HTTP response and returns processed result
-
-           Sends the query-request to the OData service, returning a client-side Enumerable for
-           subsequent in-memory operations.
-
-           Fetches HTTP response and returns processed result"""
-
+    def _build_request(self):
         if self._next_url:
             url = self._next_url
         else:
@@ -315,10 +309,46 @@ def execute(self):
         if body:
             self._logger.debug('  body: %s', body)
 
-        params = urlencode(self.get_query_params())
+        params = self.get_query_params()
+
+        return url, body, headers, params
+
+    async def async_execute(self):
+        """Fetches HTTP response and returns processed result
+
+                  Sends the query-request to the OData service, returning a client-side Enumerable for
+                  subsequent in-memory operations.
+
+                  Fetches HTTP response and returns processed result"""
+
+        url, body, headers, params = self._build_request()
+        async with self._connection.request(self.get_method(),
+                                            url,
+                                            headers=headers,
+                                            params=params,
+                                            data=body) as async_response:
+            response = ODataHttpResponse(url=async_response.url,
+                                         headers=async_response.headers,
+                                         status_code=async_response.status,
+                                         content=await async_response.read())
+        return self._call_handler(response)
+
+    def execute(self):
+        """Fetches HTTP response and returns processed result
+
+           Sends the query-request to the OData service, returning a client-side Enumerable for
+           subsequent in-memory operations.
+
+           Fetches HTTP response and returns processed result"""
+
+        url, body, headers, params = self._build_request()
+
         response = self._connection.request(
-            self.get_method(), url, headers=headers, params=params, data=body)
+            self.get_method(), url, headers=headers, params=urlencode(params), data=body)
 
+        return self._call_handler(response)
+
+    def _call_handler(self, response):
         self._logger.debug('Received response')
         self._logger.debug('  url: %s', response.url)
         self._logger.debug('  headers: %s', response.headers)
@@ -858,6 +888,19 @@ def __getattr__(self, attr):
                 raise AttributeError('EntityType {0} does not have Property {1}: {2}'
                                      .format(self._entity_type.name, attr, str(ex)))
 
+    async def async_getattr(self, attr):
+        """Get cached value of attribute or do async call to service to recover attribute value"""
+        try:
+            return self._cache[attr]
+        except KeyError:
+            try:
+                value = await self.get_proprty(attr).async_execute()
+                self._cache[attr] = value
+                return value
+            except KeyError as ex:
+                raise AttributeError('EntityType {0} does not have Property {1}: {2}'
+                                     .format(self._entity_type.name, attr, str(ex)))
+
     def nav(self, nav_property):
         """Navigates to given navigation property and returns the EntitySetProxy"""
 
@@ -1699,6 +1742,16 @@ def http_get(self, path, connection=None):
 
         return conn.get(urljoin(self._url, path))
 
+    async def async_http_get(self, path, connection=None):
+        """HTTP GET response for the passed path in the service"""
+
+        conn = connection
+        if conn is None:
+            conn = self._connection
+
+        async with conn.get(urljoin(self._url, path)) as resp:
+            return resp
+
     def http_get_odata(self, path, handler, connection=None):
         """HTTP GET request proxy for the passed path in the service"""
 
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,7 +1,9 @@
 """PyTest Fixtures"""
 import logging
 import os
+
 import pytest
+
 from pyodata.v2.model import schema_from_xml, Types
 
 
diff --git a/tests/integration/networking_libraries/test_aiohttp_client.py b/tests/integration/networking_libraries/test_aiohttp_client.py
@@ -0,0 +1,131 @@
+""" Test the pyodata integration with aiohttp client, based on asyncio
+
+https://docs.aiohttp.org/en/stable/
+"""
+import aiohttp
+from aiohttp import web
+import pytest
+
+import pyodata.v2.service
+from pyodata import Client
+from pyodata.exceptions import PyODataException, HttpError
+from pyodata.v2.model import ParserError, PolicyWarning, PolicyFatal, PolicyIgnore, Config
+
+SERVICE_URL = ''
+
+@pytest.mark.asyncio
+async def test_invalid_odata_version():
+    """Check handling of request for invalid OData version implementation"""
+
+    with pytest.raises(PyODataException) as e_info:
+        async with aiohttp.ClientSession() as client:
+            await Client.build_async_client(SERVICE_URL, client, 'INVALID VERSION')
+
+    assert str(e_info.value).startswith('No implementation for selected odata version')
+
+@pytest.mark.asyncio
+async def test_create_client_for_local_metadata(metadata):
+    """Check client creation for valid use case with local metadata"""
+
+    async with aiohttp.ClientSession() as client:
+        service_client = await Client.build_async_client(SERVICE_URL, client, metadata=metadata)
+
+        assert isinstance(service_client, pyodata.v2.service.Service)
+        assert service_client.schema.is_valid == True
+
+        assert len(service_client.schema.entity_sets) != 0
+
+@pytest.mark.asyncio
+def generate_metadata_response(headers=None, body=None, status=200):
+
+    async def metadata_response(request):
+        return web.Response(status=status, headers=headers, body=body)
+
+    return metadata_response
+
+
+@pytest.mark.parametrize("content_type", ['application/xml', 'application/atom+xml', 'text/xml'])
+@pytest.mark.asyncio
+async def test_create_service_application(aiohttp_client, metadata, content_type):
+    """Check client creation for valid MIME types"""
+
+    app = web.Application()
+    app.router.add_get('/$metadata', generate_metadata_response(headers={'content-type': content_type}, body=metadata))
+    client = await aiohttp_client(app)
+
+    service_client = await Client.build_async_client(SERVICE_URL, client)
+
+    assert isinstance(service_client, pyodata.v2.service.Service)
+
+    # one more test for '/' terminated url
+
+    service_client = await Client.build_async_client(SERVICE_URL + '/', client)
+
+    assert isinstance(service_client, pyodata.v2.service.Service)
+    assert service_client.schema.is_valid
+
+
+@pytest.mark.asyncio
+async def test_metadata_not_reachable(aiohttp_client):
+    """Check handling of not reachable service metadata"""
+
+    app = web.Application()
+    app.router.add_get('/$metadata', generate_metadata_response(headers={'content-type': 'text/html'}, status=404))
+    client = await aiohttp_client(app)
+
+    with pytest.raises(HttpError) as e_info:
+        await Client.build_async_client(SERVICE_URL, client)
+
+    assert str(e_info.value).startswith('Metadata request failed')
+
+@pytest.mark.asyncio
+async def test_metadata_saml_not_authorized(aiohttp_client):
+    """Check handling of not SAML / OAuth unauthorized response"""
+
+    app = web.Application()
+    app.router.add_get('/$metadata', generate_metadata_response(headers={'content-type': 'text/html; charset=utf-8'}))
+    client = await aiohttp_client(app)
+
+    with pytest.raises(HttpError) as e_info:
+        await Client.build_async_client(SERVICE_URL, client)
+
+    assert str(e_info.value).startswith('Metadata request did not return XML, MIME type:')
+
+
+@pytest.mark.asyncio
+async def test_client_custom_configuration(aiohttp_client, metadata):
+    """Check client creation for custom configuration"""
+
+    namespaces = {
+        'edmx': "customEdmxUrl.com",
+        'edm': 'customEdmUrl.com'
+    }
+
+    custom_config = Config(
+        xml_namespaces=namespaces,
+        default_error_policy=PolicyFatal(),
+        custom_error_policies={
+            ParserError.ANNOTATION: PolicyWarning(),
+            ParserError.ASSOCIATION: PolicyIgnore()
+        })
+
+    app = web.Application()
+    app.router.add_get('/$metadata',
+                       generate_metadata_response(headers={'content-type': 'application/xml'}, body=metadata))
+    client = await aiohttp_client(app)
+
+    with pytest.raises(PyODataException) as e_info:
+        await Client.build_async_client(SERVICE_URL, client, config=custom_config, namespaces=namespaces)
+
+    assert str(e_info.value) == 'You cannot pass namespaces and config at the same time'
+
+    with pytest.warns(DeprecationWarning,match='Passing namespaces directly is deprecated. Use class Config instead'):
+        service = await Client.build_async_client(SERVICE_URL, client, namespaces=namespaces)
+
+    assert isinstance(service, pyodata.v2.service.Service)
+    assert service.schema.config.namespaces == namespaces
+
+    service = await Client.build_async_client(SERVICE_URL, client, config=custom_config)
+
+    assert isinstance(service, pyodata.v2.service.Service)
+    assert service.schema.config == custom_config
diff --git a/tests/integration/networking_libraries/test_httpx_client_sync.py b/tests/integration/networking_libraries/test_httpx_client_sync.py
diff --git a/tests/integration/networking_libraries/test_requests_client.py b/tests/integration/networking_libraries/test_requests_client.py
