up docs and tests

Author: Mantisus

docs/guides/storage_clients.mdx

@@ -28,8 +28,8 @@ Crawlee provides three main storage client implementations:
 
 - <ApiLink to="class/FileSystemStorageClient">`FileSystemStorageClient`</ApiLink> - Provides persistent file system storage with in-memory caching.
 - <ApiLink to="class/MemoryStorageClient">`MemoryStorageClient`</ApiLink> - Stores data in memory with no persistence.
-- <ApiLink to="class/SqlStorageClient">`SqlStorageClient`</ApiLink> – Provides persistent storage using a SQL database ([SQLite](https://sqlite.org/) or [PostgreSQL](https://www.postgresql.org/)). Requires installing the extra dependency: 'crawlee[sql_sqlite]' for SQLite or 'crawlee[sql_postgres]' for PostgreSQL.
-- <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> – Provides persistent storage using a [Redis](https://redis.io/) database v8.0+. Requires installing the extra dependency `crawlee[redis]`.
+- <ApiLink to="class/SqlStorageClient">`SqlStorageClient`</ApiLink> - Provides persistent storage using a SQL database ([SQLite](https://sqlite.org/) or [PostgreSQL](https://www.postgresql.org/)). Requires installing the extra dependency: `crawlee[sql_sqlite]` for SQLite or `crawlee[sql_postgres]` for PostgreSQL.
+- <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> - Provides persistent storage using a [Redis](https://redis.io/) database v8.0+. Requires installing the extra dependency `crawlee[redis]`.
 - [`ApifyStorageClient`](https://docs.apify.com/sdk/python/reference/class/ApifyStorageClient) - Manages storage on the [Apify platform](https://apify.com), implemented in the [Apify SDK](https://github.com/apify/apify-sdk-python).
 
 ```mermaid
@@ -310,8 +310,8 @@ Configuration options for the <ApiLink to="class/SqlStorageClient">`SqlStorageCl
 
 Configuration options for the <ApiLink to="class/SqlStorageClient">`SqlStorageClient`</ApiLink> can be set via constructor arguments:
 
-- **`connection_string`** (default: SQLite in <ApiLink to="class/Configuration">`Configuration`</ApiLink> storage dir) – SQLAlchemy connection string, e.g. `sqlite+aiosqlite:///my.db` or `postgresql+asyncpg://user:pass@host/db`.
-- **`engine`** – Pre-configured SQLAlchemy AsyncEngine (optional).
+- **`connection_string`** (default: SQLite in <ApiLink to="class/Configuration">`Configuration`</ApiLink> storage dir) - SQLAlchemy connection string, e.g. `sqlite+aiosqlite:///my.db` or `postgresql+asyncpg://user:pass@host/db`.
+- **`engine`** - Pre-configured SQLAlchemy AsyncEngine (optional).
 
 For advanced scenarios, you can configure <ApiLink to="class/SqlStorageClient">`SqlStorageClient`</ApiLink> with a custom SQLAlchemy engine and additional options via the <ApiLink to="class/Configuration">`Configuration`</ApiLink> class. This is useful, for example, when connecting to an external PostgreSQL database or customizing connection pooling.
 
@@ -369,12 +369,12 @@ class RedisDatasetClient {
 %% Dataset Keys
 %% ========================
 
-class Dataset_Keys {
+class DatasetKeys {
     datasets:[name]:items - JSON Array
     datasets:[name]:metadata - JSON Object
 }
 
-class Datasets_Indexes {
+class DatasetsIndexes {
     datasets:id_to_name - Hash
     datasets:name_to_id - Hash
 }
@@ -383,8 +383,8 @@ class Datasets_Indexes {
 %% Client to Keys arrows
 %% ========================
 
-RedisDatasetClient --> Dataset_Keys
-RedisDatasetClient --> Datasets_Indexes
+RedisDatasetClient --> DatasetKeys
+RedisDatasetClient --> DatasetsIndexes
 ```
 
 ```mermaid
@@ -408,13 +408,13 @@ class RedisKeyValueStoreClient {
 %% Key-Value Store Keys
 %% ========================
 
-class Key_Value_Store_Keys {
+class KeyValueStoreKeys {
     key_value_stores:[name]:items - Hash
     key_value_stores:[name]:metadata_items - Hash
     key_value_stores:[name]:metadata - JSON Object
 }
 
-class Key_Value_Stores_Indexes {
+class KeyValueStoresIndexes {
     key_value_stores:id_to_name - Hash
     key_value_stores:name_to_id - Hash
 }
@@ -423,8 +423,8 @@ class Key_Value_Stores_Indexes {
 %% Client to Keys arrows
 %% ========================
 
-RedisKeyValueStoreClient --> Key_Value_Store_Keys
-RedisKeyValueStoreClient --> Key_Value_Stores_Indexes
+RedisKeyValueStoreClient --> KeyValueStoreKeys
+RedisKeyValueStoreClient --> KeyValueStoresIndexes
 ```
 
 ```mermaid
@@ -448,7 +448,7 @@ class RedisRequestQueueClient {
 %% Request Queue Keys
 %% ========================
 
-class Request_Queue_Keys{
+class RequestQueueKeys{
     request_queues:[name]:queue - List
     request_queues:[name]:data - Hash
     request_queues:[name]:in_progress - Hash
@@ -459,7 +459,7 @@ class Request_Queue_Keys{
     request_queues:[name]:metadata - JSON Object
 }
 
-class Request_Queues_Indexes {
+class RequestQueuesIndexes {
     request_queues:id_to_name - Hash
     request_queues:name_to_id - Hash
 }
@@ -468,8 +468,8 @@ class Request_Queues_Indexes {
 %% Client to Keys arrows
 %% ========================
 
-RedisRequestQueueClient --> Request_Queue_Keys
-RedisRequestQueueClient --> Request_Queues_Indexes
+RedisRequestQueueClient --> RequestQueueKeys
+RedisRequestQueueClient --> RequestQueuesIndexes
 ```
 
 Configuration options for the <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> can be set through environment variables or the <ApiLink to="class/Configuration">`Configuration`</ApiLink> class:
@@ -478,8 +478,8 @@ Configuration options for the <ApiLink to="class/RedisStorageClient">`RedisStora
 
 Configuration options for the <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> can be set via constructor arguments:
 
-- **`connection_string`** – Redis connection string, e.g. `redis://localhost:6379/0`.
-- **`redis`** – Pre-configured Redis client instance (optional).
+- **`connection_string`** - Redis connection string, e.g. `redis://localhost:6379/0`.
+- **`redis`** - Pre-configured Redis client instance (optional).
 
 <CodeBlock className="language-python" language="python">
     {RedisStorageClientConfigurationExample}

tests/unit/storage_clients/_redis/test_redis_dataset_client.py

@@ -144,36 +144,3 @@ async def test_metadata_record_updates(dataset_client: RedisDatasetClient) -> No
     assert metadata.created_at == initial_created
     assert metadata.modified_at > initial_modified
     assert metadata.accessed_at > accessed_after_get
-
-    # Verify metadata file is updated in Redis
-    metadata_json = await await_redis_response(dataset_client.redis.json().get(f'datasets:{metadata.name}:metadata'))
-
-    assert isinstance(metadata_json, dict)
-    assert metadata_json['item_count'] == 1  # type: ignore[unreachable] # py-json typing is broken
-
-
-@pytest.mark.usefixtures('suppress_user_warning')
-async def test_data_persistence_across_reopens(redis_client: FakeAsyncRedis) -> None:
-    """Test that data persists correctly when reopening the same dataset."""
-    storage_client = RedisStorageClient(redis=redis_client)
-
-    # Create dataset and add data
-    original_client = await storage_client.create_dataset_client(
-        name='persistence-test',
-    )
-
-    test_data = {'test_item': 'test_value', 'id': 123}
-    await original_client.push_data(test_data)
-
-    dataset_id = (await original_client.get_metadata()).id
-
-    # Reopen by ID and verify data persists
-    reopened_client = await storage_client.create_dataset_client(
-        id=dataset_id,
-    )
-
-    data = await reopened_client.get_data()
-    assert len(data.items) == 1
-    assert data.items[0] == test_data
-
-    await reopened_client.drop()

tests/unit/storage_clients/_redis/test_redis_kvs_client.py

@@ -215,31 +215,3 @@ async def test_metadata_record_updates(kvs_client: RedisKeyValueStoreClient) ->
     assert metadata.created_at == initial_created
     assert metadata.modified_at > initial_modified
     assert metadata.accessed_at > accessed_after_read
-
-
-@pytest.mark.usefixtures('suppress_user_warning')
-async def test_data_persistence_across_reopens(redis_client: FakeAsyncRedis) -> None:
-    """Test that data persists correctly when reopening the same KVS."""
-    storage_client = RedisStorageClient(redis=redis_client)
-
-    # Create KVS and add data
-    original_client = await storage_client.create_kvs_client(
-        name='persistence-test',
-    )
-
-    test_key = 'persistent-key'
-    test_value = 'persistent-value'
-    await original_client.set_value(key=test_key, value=test_value)
-
-    kvs_id = (await original_client.get_metadata()).id
-
-    # Reopen by ID and verify data persists
-    reopened_client = await storage_client.create_kvs_client(
-        id=kvs_id,
-    )
-
-    record = await reopened_client.get_value(key=test_key)
-    assert record is not None
-    assert record.value == test_value
-
-    await reopened_client.drop()

tests/unit/storage_clients/_redis/test_redis_rq_client.py

@@ -199,48 +199,6 @@ async def test_metadata_file_updates(rq_client: RedisRequestQueueClient) -> None
     assert metadata.modified_at > initial_modified
     assert metadata.accessed_at > accessed_after_read
 
-    # Verify metadata file is updated in Redis
-    metadata_json = await await_redis_response(rq_client.redis.json().get('request_queues:test_request_queue:metadata'))
-    assert isinstance(metadata_json, dict)
-    assert metadata_json['total_request_count'] == 1  # type: ignore[unreachable] # py-json typing is broken
-
-
-@pytest.mark.usefixtures('suppress_user_warning')
-async def test_data_persistence_across_reopens(redis_client: FakeAsyncRedis) -> None:
-    """Test that requests persist correctly when reopening the same RQ."""
-    storage_client = RedisStorageClient(redis=redis_client)
-
-    # Create RQ and add requests
-    original_client = await storage_client.create_rq_client(
-        name='persistence-test',
-    )
-
-    test_requests = [
-        Request.from_url('https://example.com/1'),
-        Request.from_url('https://example.com/2'),
-    ]
-    await original_client.add_batch_of_requests(test_requests)
-
-    rq_id = (await original_client.get_metadata()).id
-
-    # Reopen by ID and verify requests persist
-    reopened_client = await storage_client.create_rq_client(
-        id=rq_id,
-    )
-
-    metadata = await reopened_client.get_metadata()
-    assert metadata.total_request_count == 2
-
-    # Fetch requests to verify they're still there
-    request1 = await reopened_client.fetch_next_request()
-    request2 = await reopened_client.fetch_next_request()
-
-    assert request1 is not None
-    assert request2 is not None
-    assert {request1.url, request2.url} == {'https://example.com/1', 'https://example.com/2'}
-
-    await reopened_client.drop()
-
 
 async def test_get_request(rq_client: RedisRequestQueueClient) -> None:
     """Test that get_request works correctly."""
@@ -262,3 +220,33 @@ async def test_get_request(rq_client: RedisRequestQueueClient) -> None:
     # Test fetching a non-existent request
     non_existent = await rq_client.get_request('non-existent-id')
     assert non_existent is None
+
+
+async def test_deduplication(rq_client: RedisRequestQueueClient) -> None:
+    """Test that request deduplication works correctly."""
+    requests = [
+        Request.from_url('https://example.com/1'),
+        Request.from_url('https://example.com/1'),
+        Request.from_url('https://example.com/3'),
+    ]
+
+    await rq_client.add_batch_of_requests(requests)
+
+    # Verify only unique requests are added
+    metadata = await rq_client.get_metadata()
+    assert metadata.pending_request_count == 2
+    assert metadata.total_request_count == 2
+
+    # Fetch requests and verify order
+    request1 = await rq_client.fetch_next_request()
+    assert request1 is not None
+    assert request1 == requests[0]
+
+    # Fetch the next request, which should skip the duplicate
+    request2 = await rq_client.fetch_next_request()
+    assert request2 is not None
+    assert request2 == requests[2]
+
+    # Verify no more requests are available
+    request3 = await rq_client.fetch_next_request()
+    assert request3 is None

tests/unit/storages/conftest.py

@@ -17,7 +17,7 @@
     from fakeredis import FakeAsyncRedis
 
 
-@pytest.fixture(params=['memory', 'file_system', 'sql', 'redis_default', 'redis_bloom'])
+@pytest.fixture(params=['memory', 'file_system', 'sql', 'redis'])
 def storage_client(
     request: pytest.FixtureRequest,
     redis_client: FakeAsyncRedis,
@@ -31,10 +31,8 @@ def storage_client(
         storage_client = MemoryStorageClient()
     elif storage_type == 'sql':
         storage_client = SqlStorageClient()
-    elif storage_type == 'redis_default':
-        storage_client = RedisStorageClient(redis=redis_client, queue_dedup_strategy='default')
-    elif storage_type == 'redis_bloom':
-        storage_client = RedisStorageClient(redis=redis_client, queue_dedup_strategy='bloom')
+    elif storage_type == 'redis':
+        storage_client = RedisStorageClient(redis=redis_client)
     else:
         storage_client = FileSystemStorageClient()
     service_locator.set_storage_client(storage_client)