Support for Python 3.11 (#181)

* Testing Python 3.11
* Switching to `httptools.parser.HttpRequestParser`.
* Disabling tests for `pook` when testing Python 3.11
* Removing DeprecationWarning all over the place.
* Python 3.11 needs an async decorator.
* Adding Redis as a service container.
* Removing `async-timeout` dependency.
* Refactoring using `event_loop` fixture.
* Refactoring using `tempfile` as a context manager.
* Mark `aiohttp` tests as failing on Py3.11 - first time ever - and add a similar test with `httpx`.
* Bump version.

Author: mindflayer

.github/workflows/main.yml

@@ -15,11 +15,25 @@ concurrency:
 
 jobs:
   build:
-
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: ['3.5', '3.6', '3.7', '3.8', '3.9', '3.10', 'pypy3.9']
+        python-version: ['3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', 'pypy3.9']
+
+    services:
+      # https://docs.github.com/en/actions/using-containerized-services/about-service-containers
+      redis:
+        # Docker Hub image
+        image: redis
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps port 6379 on service container to the host
+          - 6379:6379
 
     steps:
     - uses: actions/checkout@v3
@@ -31,9 +45,6 @@ jobs:
         cache-dependency-path: |
           Pipfile
           setup.py
-    - name: Install Redis
-      run: |
-        sudo apt install redis-server
     - name: Install dependencies
       run: |
         make develop

Makefile

@@ -2,11 +2,11 @@
 
 install-dev-requirements:
 	pip install -U pip
-	pip install pipenv
+	pip install pipenv pre-commit
 
 install-test-requirements:
 	pipenv install --dev
-	pipenv run python -c "import pipfile; pf = pipfile.load('Pipfile'); print('\n'.join(package+version for package, version in pf.data['default'].items()))" > requirements.txt
+	pipenv run python -c "import pipfile; pf = pipfile.load('Pipfile'); print('\n'.join(package+version if version != '*' else package for package, version in pf.data['default'].items()))" > requirements.txt
 
 test-python:
 	@echo "Running Python tests"
@@ -34,7 +34,7 @@ publish: install-test-requirements
 	pipenv run anaconda upload dist/mocket-$(shell python -c 'import mocket; print(mocket.__version__)').tar.gz
 
 clean:
-	rm -rf *.egg-info dist/
+	rm -rf *.egg-info dist/ requirements.txt Pipfile.lock
 	find . -type d -name __pycache__ -exec rm -rf {} \;
 
 .PHONY: clean publish safetest test setup develop lint-python test-python install-test-requirements install-dev-requirements

Pipfile

@@ -7,13 +7,14 @@ name = "pypi"
 python-magic = ">=0.4.5"
 decorator = ">=4.0.0"
 urllib3 = ">=1.25.3"
-http-parser = ">=0.9.0"
+httptools = "*"
 
 [dev-packages]
 pre-commit = "*"
-pytest = ">4.6"
+pytest = "*"
 pytest-cov = "*"
 pytest-asyncio = "*"
+asgiref = "*"
 requests = "*"
 redis = "*"
 gevent = "*"
@@ -22,7 +23,6 @@ pook = "*"
 flake8 = "<7"
 xxhash = "*"
 aiohttp = "*"
-async-timeout = "*"
 httpx = "*"
 pipfile = "*"
 build = "*"

README.rst

@@ -231,7 +231,6 @@ Example:
 
     import aiohttp
     import asyncio
-    import async_timeout
     from unittest import TestCase
 
     from mocket.plugins.httpretty import httpretty, httprettified
@@ -248,13 +247,14 @@ Example:
             )
 
             async def main(l):
-                async with aiohttp.ClientSession(loop=l) as session:
-                    with async_timeout.timeout(3):
-                        async with session.get(url) as get_response:
-                            assert get_response.status == 200
-                            assert await get_response.text() == '{"origin": "127.0.0.1"}'
+                async with aiohttp.ClientSession(
+                    loop=l, timeout=aiohttp.ClientTimeout(total=3)
+                ) as session:
+                    async with session.get(url) as get_response:
+                        assert get_response.status == 200
+                        assert await get_response.text() == '{"origin": "127.0.0.1"}'
 
-            loop = asyncio.get_event_loop()
+            loop = asyncio.new_event_loop()
             loop.set_debug(True)
             loop.run_until_complete(main(loop))
 
@@ -277,18 +277,18 @@ Example:
             Entry.single_register(Entry.POST, url, body=body*2, status=201)
 
             async def main(l):
-                async with aiohttp.ClientSession(loop=l) as session:
-                    with async_timeout.timeout(3):
-                        async with session.get(url) as get_response:
-                            assert get_response.status == 404
-                            assert await get_response.text() == body
-
-                    with async_timeout.timeout(3):
-                        async with session.post(url, data=body * 6) as post_response:
-                            assert post_response.status == 201
-                            assert await post_response.text() == body * 2
-
-            loop = asyncio.get_event_loop()
+                async with aiohttp.ClientSession(
+                    loop=l, timeout=aiohttp.ClientTimeout(total=3)
+                ) as session:
+                    async with session.get(url) as get_response:
+                        assert get_response.status == 404
+                        assert await get_response.text() == body
+
+                    async with session.post(url, data=body * 6) as post_response:
+                        assert post_response.status == 201
+                        assert await post_response.text() == body * 2
+
+            loop = asyncio.new_event_loop()
             loop.run_until_complete(main(loop))
 
     # or again with a unittest.IsolatedAsyncioTestCase
@@ -302,18 +302,18 @@ Example:
             Entry.single_register(Entry.GET, url, body=body, status=404)
             Entry.single_register(Entry.POST, url, body=body * 2, status=201)
 
-            async with aiohttp.ClientSession() as session:
-                with async_timeout.timeout(3):
-                    async with session.get(url) as get_response:
-                        assert get_response.status == 404
-                        assert await get_response.text() == body
-
-                with async_timeout.timeout(3):
-                    async with session.post(url, data=body * 6) as post_response:
-                        assert post_response.status == 201
-                        assert await post_response.text() == body * 2
-                        assert Mocket.last_request().method == 'POST'
-                        assert Mocket.last_request().body == body * 6
+            async with aiohttp.ClientSession(
+                timeout=aiohttp.ClientTimeout(total=3)
+            ) as session:
+                async with session.get(url) as get_response:
+                    assert get_response.status == 404
+                    assert await get_response.text() == body
+
+                async with session.post(url, data=body * 6) as post_response:
+                    assert post_response.status == 201
+                    assert await post_response.text() == body * 2
+                    assert Mocket.last_request().method == 'POST'
+                    assert Mocket.last_request().body == body * 6
 
 
 Works well with others

mocket/__init__.py

@@ -3,4 +3,4 @@
 
 __all__ = ("async_mocketize", "mocketize", "Mocket", "MocketEntry", "Mocketizer")
 
-__version__ = "3.10.9"
+__version__ = "3.11.0"

mocket/mockhttp.py

@@ -3,13 +3,10 @@
 from http.server import BaseHTTPRequestHandler
 from urllib.parse import parse_qs, unquote, urlsplit
 
-from .compat import decode_from_bytes, do_the_magic, encode_to_bytes
-from .mocket import Mocket, MocketEntry
+from httptools.parser import HttpRequestParser
 
-try:
-    from http_parser.parser import HttpParser
-except ImportError:
-    from http_parser.pyparser import HttpParser
+from .compat import ENCODING, decode_from_bytes, do_the_magic, encode_to_bytes
+from .mocket import Mocket, MocketEntry
 
 try:
     import magic
@@ -21,31 +18,59 @@
 CRLF = "\r\n"
 
 
+class Protocol:
+    def __init__(self):
+        self.url = None
+        self.body = None
+        self.headers = {}
+
+    def on_header(self, name: bytes, value: bytes):
+        self.headers[name.decode("ascii")] = value.decode("ascii")
+
+    def on_body(self, body: bytes):
+        try:
+            self.body = body.decode(ENCODING)
+        except UnicodeDecodeError:
+            self.body = body
+
+    def on_url(self, url: bytes):
+        self.url = url.decode("ascii")
+
+
 class Request:
-    parser = None
-    _body = None
+    _protocol = None
+    _parser = None
 
     def __init__(self, data):
-        self.parser = HttpParser()
-        self.parser.execute(data, len(data))
-
-        self.method = self.parser.get_method()
-        self.path = self.parser.get_path()
-        self.headers = self.parser.get_headers()
-        self.querystring = parse_qs(
-            unquote(self.parser.get_query_string()), keep_blank_values=True
-        )
-        if self.querystring:
-            self.path += "?{}".format(self.parser.get_query_string())
+        self._protocol = Protocol()
+        self._parser = HttpRequestParser(self._protocol)
+        self.add_data(data)
 
     def add_data(self, data):
-        self.parser.execute(data, len(data))
+        self._parser.feed_data(data)
+
+    @property
+    def method(self):
+        return self._parser.get_method().decode("ascii")
+
+    @property
+    def path(self):
+        return self._protocol.url
+
+    @property
+    def headers(self):
+        return self._protocol.headers
+
+    @property
+    def querystring(self):
+        parts = self._protocol.url.split("?", 1)
+        if len(parts) == 2:
+            return parse_qs(unquote(parts[1]), keep_blank_values=True)
+        return {}
 
     @property
     def body(self):
-        if self._body is None:
-            self._body = decode_from_bytes(self.parser.recv_body())
-        return self._body
+        return self._protocol.body
 
     def __str__(self):
         return "{} - {} - {}".format(self.method, self.path, self.headers)

mocket/plugins/httpretty/__init__.py

@@ -1,6 +1,6 @@
 from mocket import Mocket, mocketize
 from mocket.async_mocket import async_mocketize
-from mocket.compat import byte_type, text_type
+from mocket.compat import ENCODING, byte_type, text_type
 from mocket.mockhttp import Entry as MocketHttpEntry
 from mocket.mockhttp import Request as MocketHttpRequest
 from mocket.mockhttp import Response as MocketHttpResponse
@@ -13,9 +13,11 @@ def httprettifier_headers(headers):
 class Request(MocketHttpRequest):
     @property
     def body(self):
-        if self._body is None:
-            self._body = self.parser.recv_body()
-        return self._body
+        return super().body.encode(ENCODING)
+
+    @property
+    def headers(self):
+        return httprettifier_headers(super().headers)
 
 
 class Response(MocketHttpResponse):
@@ -116,6 +118,7 @@ def __getattr__(self, name):
 
 __all__ = (
     "HTTPretty",
+    "httpretty",
     "activate",
     "async_httprettified",
     "httprettified",