Merge branch 'unstable'

Author: Grokzen

.travis.yml

@@ -9,19 +9,19 @@ python:
 services:
   - redis-server
 install:
-  - "if [[ $REDIS_VERSION == '3.0' ]]; then REDIS_VERSION=3.0.7 make redis-install; fi"
-  - "if [[ $REDIS_VERSION == '3.2' ]]; then REDIS_VERSION=3.2.0-rc3 make redis-install; fi"
+  - "if [[ $REDIS_VERSION == '3.0' ]]; then REDIS_VERSION=3.0 make redis-install; fi"
+  - "if [[ $REDIS_VERSION == '3.2' ]]; then REDIS_VERSION=3.2 make redis-install; fi"
   - pip install -r dev-requirements.txt
   - pip install -e .
   - "if [[ $HIREDIS == '1' ]]; then pip install hiredis; fi"
 env:
-  # Redis 3.0.7
+  # Redis 3.0
   - HIREDIS=0 REDIS_VERSION=3.0
-  # Redis 3.0.7 and HIREDIS
+  # Redis 3.0 and HIREDIS
   - HIREDIS=1 REDIS_VERSION=3.0
-  # Redis 3.2.0-rc3
+  # Redis 3.2
   - HIREDIS=0 REDIS_VERSION=3.2
-  # Redis 3.2.0-rc3 and HIREDIS
+  # Redis 3.2 and HIREDIS
   - HIREDIS=1 REDIS_VERSION=3.2
 script: 
   - make start

docs/authors.rst

@@ -20,3 +20,4 @@ Authors who contributed code or testing:
  - gmolight - https://github.com/gmolight
  - baranbartu - https://github.com/baranbartu
  - monklof - https://github.com/monklof
+ - dutradda - https://github.com/dutradda

docs/release-notes.rst

@@ -1,6 +1,15 @@
 Release Notes
 =============
 
+1.3.2 (Nov 27, 2016)
+--------------------
+
+    * Fix a bug where from_url was not possible to use without passing in additional variables. Now it works as the same method from redis-py.
+      Note that the same rules that is currently in place for passing ip addresses/dns names into startup_nodes variable apply the same way through
+      the from_url method.
+    * Added options to skip full coverage check. This flag is useful when the CONFIG redis command is disabled by the server.
+    * Fixed a bug where method *CLUSTER SLOTS* would break in newer redis versions where node id is included in the reponse. Method is not compatible with both old and new redis versions.
+
 
 1.3.1 (Oct 13, 2016)
 --------------------

docs/upgrading.rst

@@ -3,6 +3,14 @@ Upgrading redis-py-cluster
 
 This document describes what must be done when upgrading between different versions to ensure that code still works.
 
+
+1.3.1 --> 1.3.2
+---------------
+
+If your redis instance is configured to not have the `CONFIG ...` comannds enabled due to security reasons you need to pass this into the client object `skip_full_coverage_check=True`. Benefits is that the client class no longer requires the `CONFIG ...` commands to be enabled on the server. Downsides is that you can't use the option in your redis server and still use the same feature in this client.
+
+
+
 1.3.0 --> 1.3.1
 ---------------
 

ptp-debug.py

@@ -4,5 +4,6 @@
 
 # Note: decode_responses must be set to True when used with python3
 rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
+url_client = StrictRedisCluster.from_url('http://127.0.0.1:7000')
 
 __import__('ptpdb').set_trace()

rediscluster/__init__.py

@@ -16,7 +16,7 @@
 setattr(redis, "StrictClusterPipeline", StrictClusterPipeline)
 
 # Major, Minor, Fix version
-__version__ = (1, 3, 1)
+__version__ = (1, 3, 2)
 
 if sys.version_info[0:3] == (3, 4, 0):
     raise RuntimeError("CRITICAL: rediscluster do not work with python 3.4.0. Please use 3.4.1 or higher.")

rediscluster/client.py

@@ -113,7 +113,7 @@ class StrictRedisCluster(StrictRedis):
     }
 
     def __init__(self, host=None, port=None, startup_nodes=None, max_connections=32, max_connections_per_node=False, init_slot_cache=True,
-                 readonly_mode=False, reinitialize_steps=None, **kwargs):
+                 readonly_mode=False, reinitialize_steps=None, skip_full_coverage_check=False, **kwargs):
         """
         :startup_nodes:
             List of nodes that initial bootstrapping can be done from
@@ -125,6 +125,9 @@ def __init__(self, host=None, port=None, startup_nodes=None, max_connections=32,
             Maximum number of connections that should be kept open at one time
         :readonly_mode:
             enable READONLY mode. You can read possibly stale data from slave.
+        :skip_full_coverage_check:
+            Skips the check of cluster-require-full-coverage config, useful for clusters
+            without the CONFIG command (like aws)
         :**kwargs:
             Extra arguments that will be sent into StrictRedis instance when created
             (See Official redis-py doc for supported kwargs
@@ -156,6 +159,7 @@ def __init__(self, host=None, port=None, startup_nodes=None, max_connections=32,
                 max_connections=max_connections,
                 reinitialize_steps=reinitialize_steps,
                 max_connections_per_node=max_connections_per_node,
+                skip_full_coverage_check=skip_full_coverage_check,
                 **kwargs
             )
 
@@ -167,6 +171,30 @@ def __init__(self, host=None, port=None, startup_nodes=None, max_connections=32,
         self.response_callbacks = self.__class__.RESPONSE_CALLBACKS.copy()
         self.response_callbacks = dict_merge(self.response_callbacks, self.CLUSTER_COMMANDS_RESPONSE_CALLBACKS)
 
+    @classmethod
+    def from_url(cls, url, db=None, skip_full_coverage_check=False, **kwargs):
+        """
+        Return a Redis client object configured from the given URL, which must
+        use either `the ``redis://`` scheme
+        <http://www.iana.org/assignments/uri-schemes/prov/redis>`_ for RESP
+        connections or the ``unix://`` scheme for Unix domain sockets.
+        For example::
+            redis://[:password]@localhost:6379/0
+            unix://[:password]@/path/to/socket.sock?db=0
+        There are several ways to specify a database number. The parse function
+        will return the first specified option:
+            1. A ``db`` querystring option, e.g. redis://localhost?db=0
+            2. If using the redis:// scheme, the path argument of the url, e.g.
+               redis://localhost/0
+            3. The ``db`` argument to this function.
+        If none of these options are specified, db=0 is used.
+        Any additional querystring arguments and keyword arguments will be
+        passed along to the ConnectionPool class's initializer. In the case
+        of conflicting arguments, querystring arguments always win.
+        """
+        connection_pool = ClusterConnectionPool.from_url(url, db=db, **kwargs)
+        return cls(connection_pool=connection_pool, skip_full_coverage_check=skip_full_coverage_check)
+
     def __repr__(self):
         """
         """

rediscluster/connection.py

@@ -70,15 +70,32 @@ class ClusterConnectionPool(ConnectionPool):
     RedisClusterDefaultTimeout = None
 
     def __init__(self, startup_nodes=None, init_slot_cache=True, connection_class=ClusterConnection,
-                 max_connections=None, max_connections_per_node=False, reinitialize_steps=None, **connection_kwargs):
+                 max_connections=None, max_connections_per_node=False, reinitialize_steps=None,
+                 skip_full_coverage_check=False, **connection_kwargs):
         """
+        :skip_full_coverage_check:
+            Skips the check of cluster-require-full-coverage config, useful for clusters
+            without the CONFIG command (like aws)
         """
         super(ClusterConnectionPool, self).__init__(connection_class=connection_class, max_connections=max_connections)
 
+        # Special case to make from_url method compliant with cluster setting.
+        # from_url method will send in the ip and port through a different variable then the
+        # regular startup_nodes variable.
+        if startup_nodes is None:
+            if 'port' in connection_kwargs and 'host' in connection_kwargs:
+                startup_nodes = [{
+                    'host': connection_kwargs.pop('host'),
+                    'port': str(connection_kwargs.pop('port')),
+                }]
+
+        print(startup_nodes)
+
         self.max_connections = max_connections or 2 ** 31
         self.max_connections_per_node = max_connections_per_node
 
-        self.nodes = NodeManager(startup_nodes, reinitialize_steps=reinitialize_steps, **connection_kwargs)
+        self.nodes = NodeManager(startup_nodes, reinitialize_steps=reinitialize_steps,
+                                 skip_full_coverage_check=skip_full_coverage_check, **connection_kwargs)
         if init_slot_cache:
             self.nodes.initialize()
 

rediscluster/nodemanager.py

@@ -19,8 +19,11 @@ class NodeManager(object):
     """
     RedisClusterHashSlots = 16384
 
-    def __init__(self, startup_nodes=None, reinitialize_steps=None, **connection_kwargs):
+    def __init__(self, startup_nodes=None, reinitialize_steps=None, skip_full_coverage_check=False, **connection_kwargs):
         """
+        :skip_full_coverage_check:
+            Skips the check of cluster-require-full-coverage config, useful for clusters
+            without the CONFIG command (like aws)
         """
         self.connection_kwargs = connection_kwargs
         self.nodes = {}
@@ -29,6 +32,7 @@ def __init__(self, startup_nodes=None, reinitialize_steps=None, **connection_kwa
         self.orig_startup_nodes = [node for node in self.startup_nodes]
         self.reinitialize_counter = 0
         self.reinitialize_steps = reinitialize_steps or 25
+        self._skip_full_coverage_check = skip_full_coverage_check
 
         if not self.startup_nodes:
             raise RedisClusterException("No startup nodes provided")
@@ -218,7 +222,10 @@ def initialize(self):
                 self.populate_startup_nodes()
                 self.refresh_table_asap = False
 
-            need_full_slots_coverage = self.cluster_require_full_coverage(nodes_cache)
+            if self._skip_full_coverage_check:
+                need_full_slots_coverage = False
+            else:
+                need_full_slots_coverage = self.cluster_require_full_coverage(nodes_cache)
 
             # Validate if all slots are covered or if we should try next startup node
             for i in range(0, self.RedisClusterHashSlots):

rediscluster/utils.py

@@ -125,8 +125,8 @@ def parse_cluster_slots(resp, **options):
     """
     current_host = options.get('current_host', '')
 
-    def fix_server(host, port):
-        return (host or current_host, port)
+    def fix_server(*args):
+        return (args[0] or current_host, args[1])
 
     slots = {}
     for slot in resp:

setup.py

@@ -20,7 +20,7 @@
 
 setup(
     name="redis-py-cluster",
-    version="1.3.1",
+    version="1.3.2",
     description="Cluster library for redis 3.0.0 built on top of redis-py lib",
     long_description=readme + '\n\n' + history,
     author="Johan Andersson",

tests/test_cluster_obj.py

@@ -15,8 +15,9 @@
 from tests.conftest import _get_client, skip_if_server_version_lt, skip_if_not_password_protected_nodes
 
 # 3rd party imports
-from mock import patch, Mock
+from mock import patch, Mock, MagicMock
 from redis._compat import b, unicode
+from redis import StrictRedis
 import pytest
 
 pytestmark = skip_if_server_version_lt('2.9.0')
@@ -107,6 +108,17 @@ def test_custom_connectionpool():
     assert {"host": h, "port": p} in c.connection_pool.nodes.startup_nodes
 
 
+@patch('rediscluster.nodemanager.StrictRedis', new=MagicMock())
+def test_skip_full_coverage_check():
+    """
+    Test if the cluster_require_full_coverage NodeManager method was not called with the flag activated
+    """
+    c = StrictRedisCluster("192.168.0.1", 7001, init_slot_cache=False, skip_full_coverage_check=True)
+    c.connection_pool.nodes.cluster_require_full_coverage = MagicMock()
+    c.connection_pool.nodes.initialize()
+    assert not c.connection_pool.nodes.cluster_require_full_coverage.called
+
+
 def test_blocked_commands(r):
     """
     These commands should be blocked and raise RedisClusterException

tests/test_utils.py

@@ -14,13 +14,63 @@
     merge_result,
     first_key,
     clusterdown_wrapper,
+    parse_cluster_slots,
 )
 
 # 3rd party imports
 import pytest
 from redis._compat import unicode
 
 
+def test_parse_cluster_slots():
+    """
+    Example raw output from redis cluster. Output is form a redis 3.2.x node
+    that includes the id in the reponse. The test below that do not include the id
+    is to validate that the code is compatible with redis versions that do not contain
+    that value in the response from the server.
+
+    127.0.0.1:10000> cluster slots
+    1) 1) (integer) 5461
+       2) (integer) 10922
+       3) 1) "10.0.0.1"
+          2) (integer) 10000
+          3) "3588b4cf9fc72d57bb262a024747797ead0cf7ea"
+       4) 1) "10.0.0.4"
+          2) (integer) 10000
+          3) "a72c02c7d85f4ec3145ab2c411eefc0812aa96b0"
+    2) 1) (integer) 10923
+       2) (integer) 16383
+       3) 1) "10.0.0.2"
+          2) (integer) 10000
+          3) "ffd36d8d7cb10d813f81f9662a835f6beea72677"
+       4) 1) "10.0.0.5"
+          2) (integer) 10000
+          3) "5c15b69186017ddc25ebfac81e74694fc0c1a160"
+    3) 1) (integer) 0
+       2) (integer) 5460
+       3) 1) "10.0.0.3"
+          2) (integer) 10000
+          3) "069cda388c7c41c62abe892d9e0a2d55fbf5ffd5"
+       4) 1) "10.0.0.6"
+          2) (integer) 10000
+          3) "dc152a08b4cf1f2a0baf775fb86ad0938cb907dc"
+    """
+    mock_response = [
+        [0, 5460, ['172.17.0.2', 7000], ['172.17.0.2', 7003]],
+        [5461, 10922, ['172.17.0.2', 7001], ['172.17.0.2', 7004]],
+        [10923, 16383, ['172.17.0.2', 7002], ['172.17.0.2', 7005]]
+    ]
+    parse_cluster_slots(mock_response)
+
+    extended_mock_response = [
+        [0, 5460, ['172.17.0.2', 7000, 'ffd36d8d7cb10d813f81f9662a835f6beea72677'], ['172.17.0.2', 7003, '5c15b69186017ddc25ebfac81e74694fc0c1a160']],
+        [5461, 10922, ['172.17.0.2', 7001, '069cda388c7c41c62abe892d9e0a2d55fbf5ffd5'], ['172.17.0.2', 7004, 'dc152a08b4cf1f2a0baf775fb86ad0938cb907dc']],
+        [10923, 16383, ['172.17.0.2', 7002, '3588b4cf9fc72d57bb262a024747797ead0cf7ea'], ['172.17.0.2', 7005, 'a72c02c7d85f4ec3145ab2c411eefc0812aa96b0']]
+    ]
+
+    parse_cluster_slots(extended_mock_response)
+
+
 def test_string_keys_to():
     def mock_true():
         return True