Improve import from RDB files

- Use custom rdbtools to speed up keys scans
- Improve validation for inputs and error handling

Author: uglide

src/modules/bulk-operations/operations/rdbimport.cpp

@@ -18,16 +18,30 @@ BulkOperations::RDBImportOperation::RDBImportOperation(
 
 void BulkOperations::RDBImportOperation::getAffectedKeys(
     std::function<void(QVariant, QString)> callback) {
+
+  m_keyPattern.setPatternSyntax(QRegExp::RegExp2);
+
+  if (!m_keyPattern.isValid()) {
+    return callback(QVariant(), QCoreApplication::translate(
+                                    "RDM", "Invalid regexp for keys filter."));
+  }
+
   m_python->call_native(
       "rdb.rdb_list_keys",
       QVariantList{m_metadata["path"].toString(), m_metadata["db"].toInt(),
                    m_keyPattern.pattern()},
       [callback, this](QVariant v) {
         m_affectedKeys.clear();
 
+        if (v.isNull()) {
+          return callback(QVariant(),
+                          QCoreApplication::translate(
+                              "RDM", "Cannot get the list of affected keys"));
+        }
+
         QVariantList keys = v.toList();
 
-        for (QVariant k : keys) {
+        for (const QVariant &k : qAsConst(keys)) {
           m_affectedKeys.append(QString::fromUtf8(k.toByteArray()));
         }
 
@@ -45,7 +59,7 @@ QList<QByteArray> convertToByteArray(QVariant v) {
 
   QList<QByteArray> result;
 
-  for (QVariant b : l) {
+  for (const QVariant &b : qAsConst(l)) {
     result.append(b.toByteArray());
   }
 
@@ -68,7 +82,7 @@ void BulkOperations::RDBImportOperation::performOperation(
   auto processCommands = [this, returnResults](const QVariantList& commands) {
     QList<QList<QByteArray>> rawCmds;
 
-    for (QVariant cmd : commands) {
+    for (const QVariant &cmd : commands) {
       auto rawCmd = convertToByteArray(cmd);
 
       if (rawCmd.at(0).toLower() == QByteArray("select")) {

src/py/rdb/__init__.py

@@ -9,7 +9,7 @@
 def process_command(callback, path_to_rdb, db,
                     include_keys_pattern,
                     exclude_keys_pattern,
-                    key_types):
+                    key_types, scan_keys=False):
     filters = {}
 
     if db:
@@ -32,7 +32,7 @@ def process_command(callback, path_to_rdb, db,
             else:
                 filters['types'].append(x)
 
-    parser = rdbtools.RdbParser(callback=callback, filters=filters)
+    parser = rdbtools.RdbParser(callback=callback, filters=filters, ignore_values=scan_keys)
     parser.parse(path_to_rdb)
 
 
@@ -45,43 +45,9 @@ def __init__(self, string_escape=None):
             super(KeysOnlyCallback, self).__init__(string_escape)
             self._out = set()
 
-        def _keyout(self, key):
+        def key(self, key):
             self._out.add(self.encode_key(key))
 
-        def set(self, key, value, expiry, info):
-            self._keyout(key)
-
-        def start_hash(self, key, length, expiry, info):
-            self._keyout(key)
-
-        def hset(self, key, field, value):
-            self._keyout(key)
-
-        def start_set(self, key, cardinality, expiry, info):
-            self._keyout(key)
-
-        def sadd(self, key, member):
-            self._keyout(key)
-
-        def start_list(self, key, expiry, info):
-            self._keyout(key)
-
-        def rpush(self, key, value):
-            self._keyout(key)
-
-        def start_sorted_set(self, key, length, expiry, info):
-            self._keyout(key)
-
-        def zadd(self, key, score, member):
-            self._keyout(key)
-
-        def start_stream(self, key, listpacks_count, expiry, info):
-            self._keyout(key)
-
-        def start_module(self, key, module_name, expiry, info):
-            self._keyout(key)
-            return False
-
         def keys(self):
             return list(self._out)
 
@@ -217,3 +183,4 @@ def commands(self):
                     key_types)
 
     return callback.commands()
+

src/py/requirements.txt

@@ -2,5 +2,5 @@ bitstring
 cbor
 msgpack
 git+https://github.com/mrnom/phpserialize.git#egg=phpserialize
-rdbtools
+git+https://github.com/uglide/redis-rdb-tools#egg=rdbtools
 python-lzf