Adding xxhash for speeding up mocket (#84)

* Switch to xxhash when possible. (#83)
* Not ready for Pipenv, small refactor, adding xxhash to test modules.
* Adding tests for hashing with xxhash.
* Adding documentation

Author: mindflayer

README.rst

@@ -55,6 +55,12 @@ Using pip::
 
     $ pip install mocket
 
+Speedups
+========
+Mocket uses **xxhash** when available instead of *hashlib.md5* for creating hashes, you can install it as follows::
+
+    $ pip install mocket[speedups]
+
 Issues
 ============
 When opening an **Issue**, please add few lines of code as failing test, or -better- open its relative **Pull request** adding this test to our test suite.

mocket/mocket.py

@@ -1,29 +1,31 @@
 from __future__ import unicode_literals
-import socket
-import json
-import os
-import ssl
-import io
+
 import collections
 import hashlib
+import io
+import json
+import os
 import select
+import socket
+import ssl
 from datetime import datetime, timedelta
 
 import decorator
 import hexdump
 
-from .utils import (
-    MocketSocketCore,
-)
-from .compat import (
-    encode_to_bytes,
-    decode_from_bytes,
-    basestring,
-    byte_type,
-    text_type,
-    FileNotFoundError,
-    JSONDecodeError,
-)
+from .compat import (FileNotFoundError, JSONDecodeError, basestring, byte_type,
+                     decode_from_bytes, encode_to_bytes, text_type)
+from .utils import MocketSocketCore
+
+xxh32 = None
+try:
+    from xxhash import xxh32
+except ImportError:
+    try:
+        from xxhash_cffi import xxh32
+    except ImportError:
+        pass
+hasher = xxh32 or hashlib.md5
 
 try:
     from urllib3.contrib.pyopenssl import inject_into_urllib3, extract_from_urllib3
@@ -118,6 +120,10 @@ def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_ad
     return s
 
 
+def _hash_request(h, req):
+    return h(encode_to_bytes(''.join(sorted(req.split('\r\n'))))).hexdigest()
+
+
 class MocketSocket(object):
     family = None
     type = None
@@ -262,7 +268,7 @@ def _connect(self):  # pragma: no cover
     def true_sendall(self, data, *args, **kwargs):
         req = decode_from_bytes(data)
         # make request unique again
-        req_signature = hashlib.md5(encode_to_bytes(''.join(sorted(req.split('\r\n'))))).hexdigest()
+        req_signature = _hash_request(hasher, req)
         # port should be always a string
         port = text_type(self._port)
 
@@ -283,7 +289,15 @@ def true_sendall(self, data, *args, **kwargs):
                 pass
 
         try:
-            response_dict = responses[self._host][port][req_signature]
+            try:
+                response_dict = responses[self._host][port][req_signature]
+            except KeyError:
+                if hasher is not hashlib.md5:
+                    # Fallback for backwards compatibility
+                    req_signature = _hash_request(hashlib.md5, req)
+                    response_dict = responses[self._host][port][req_signature]
+                else:
+                    raise
         except KeyError:
             # preventing next KeyError exceptions
             responses.setdefault(self._host, dict())

mocket/mockhttp.py

@@ -1,23 +1,18 @@
 from __future__ import unicode_literals
+
 import re
 import time
 from io import BytesIO
 
+from .compat import (BaseHTTPRequestHandler, decode_from_bytes,
+                     encode_to_bytes, parse_qs, unquote_utf8, urlsplit)
+from .mocket import Mocket, MocketEntry
+
 try:
     import magic
 except ImportError:
     magic = None
 
-from .compat import (
-    BaseHTTPRequestHandler,
-    urlsplit,
-    parse_qs,
-    encode_to_bytes,
-    decode_from_bytes,
-    unquote_utf8,
-)
-from .mocket import Mocket, MocketEntry
-
 
 STATUS = dict([(k, v[0]) for k, v in BaseHTTPRequestHandler.responses.items()])
 CRLF = '\r\n'

mocket/mockredis.py

@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
+
 from itertools import chain
 
-from .compat import text_type, byte_type, encode_to_bytes, decode_from_bytes, shsplit
-from .mocket import MocketEntry, Mocket
+from .compat import (byte_type, decode_from_bytes, encode_to_bytes, shsplit,
+                     text_type)
+from .mocket import Mocket, MocketEntry
 
 
 class Request(object):

runtests.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
-import sys
 import os
+import sys
 
 
 def runtests(args=None):
@@ -13,11 +13,15 @@ def runtests(args=None):
 
     python35 = False
 
+    extras = ['xxhash']
+
     # aiohttp available on Python >=3.5
     if major == 3 and minor >= 5:
         python35 = True
 
-        os.system('pip install aiohttp async_timeout')
+        extras += ['aiohttp', 'async_timeout']
+
+    os.system('pip install {}'.format(' '.join(extras)))
 
     if not any(a for a in args[1:] if not a.startswith('-')):
         args.append('tests/main')

setup.py

@@ -1,8 +1,7 @@
 import io
 import sys
 
-from setuptools import setup, find_packages, os
-
+from setuptools import find_packages, os, setup
 
 major, minor = sys.version_info[:2]
 
@@ -39,6 +38,10 @@ def read_version(package):
     packages=find_packages(exclude=exclude_packages),
     install_requires=install_requires,
     extras_require={
+        'speedups': [
+            'xxhash;platform_python_implementation=="CPython"',
+            'xxhash-cffi;platform_python_implementation=="PyPy"'
+        ],
         'tests': tests_requires,
         'dev': [],
         'pook': pook_requires,  # plugins version supporting mocket.plugins.pook.MocketEngine

tests/main/test_http.py

@@ -1,20 +1,20 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
-import os
+
 import io
-import time
 import json
-import mock
+import os
 import tempfile
+import time
 from unittest import TestCase
 
+import mock
 import pytest
 import requests
+from tests import HTTPError, urlencode, urlopen
 
 from mocket import Mocket, mocketize
 from mocket.mockhttp import Entry, Response
-from tests import urlopen, urlencode, HTTPError
-
 
 recording_directory = tempfile.mkdtemp()
 
@@ -91,20 +91,6 @@ def test_truesendall_with_chunk_recording(self):
 
         assert len(responses['httpbin.org']['80'].keys()) == 1
 
-    @mocketize(truesocket_recording_dir=os.path.dirname(__file__))
-    def test_truesendall_with_dump_from_recording(self):
-        requests.get('http://httpbin.org/ip', headers={"user-agent": "Fake-User-Agent"})
-        requests.get('http://httpbin.org/gzip', headers={"user-agent": "Fake-User-Agent"})
-
-        dump_filename = os.path.join(
-            Mocket.get_truesocket_recording_dir(),
-            Mocket.get_namespace() + '.json',
-        )
-        with io.open(dump_filename) as f:
-            responses = json.load(f)
-
-        self.assertEqual(len(responses['httpbin.org']['80'].keys()), 2)
-
     @mocketize
     def test_wrongpath_truesendall(self):
         Entry.register(Entry.GET, 'http://httpbin.org/user.agent', Response(status=404))
@@ -291,6 +277,20 @@ def test_request_bodies(self):
             last_request = Mocket.last_request()
             assert last_request.body == request_body
 
+    @mocketize(truesocket_recording_dir=os.path.dirname(__file__))
+    def test_truesendall_with_dump_from_recording(self):
+        requests.get('http://httpbin.org/ip', headers={"user-agent": "Fake-User-Agent"})
+        requests.get('http://httpbin.org/gzip', headers={"user-agent": "Fake-User-Agent"})
+
+        dump_filename = os.path.join(
+            Mocket.get_truesocket_recording_dir(),
+            Mocket.get_namespace() + '.json',
+        )
+        with io.open(dump_filename) as f:
+            responses = json.load(f)
+
+        self.assertEqual(len(responses['httpbin.org']['80'].keys()), 2)
+
     # @mocketize
     # def test_http_basic_auth(self):
     #     url = 'http://httpbin.org/hidden-basic-auth/hellouser/hellopassword'

tests/main/test_http_with_xxhash.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import io
+import json
+import os
+
+import requests
+from tests.main.test_http import HttpTestCase
+
+from mocket import Mocket, mocketize
+
+
+class HttpEntryTestCase(HttpTestCase):
+
+    @mocketize(truesocket_recording_dir=os.path.dirname(__file__))
+    def test_truesendall_with_dump_from_recording(self):
+        requests.get('http://httpbin.org/ip', headers={"user-agent": "Fake-User-Agent"})
+        requests.get('http://httpbin.org/gzip', headers={"user-agent": "Fake-User-Agent"})
+
+        dump_filename = os.path.join(
+            Mocket.get_truesocket_recording_dir(),
+            Mocket.get_namespace() + '.json',
+        )
+        with io.open(dump_filename) as f:
+            responses = json.load(f)
+
+        self.assertEqual(len(responses['httpbin.org']['80'].keys()), 2)

tests/main/test_https.py

@@ -1,14 +1,14 @@
-import os
 import io
 import json
+import os
 import tempfile
 
-import requests
 import pytest
+import requests
+from tests import urlopen
 
-from mocket import mocketize, Mocket
+from mocket import Mocket, mocketize
 from mocket.mockhttp import Entry
-from tests import urlopen
 
 
 @pytest.fixture

tests/main/test_mocket.py

@@ -1,11 +1,12 @@
 from __future__ import unicode_literals
+
+import io
 import socket
 from unittest import TestCase
-import io
 
 import pytest
 
-from mocket import Mocket, mocketize, MocketEntry, Mocketizer
+from mocket import Mocket, MocketEntry, Mocketizer, mocketize
 from mocket.compat import encode_to_bytes