Additional testing for AsyncArray, Array (#3049)

* remove duplicate metadata parsing

* add test cases

* add test cases

* tests for different zarr_formats in test_storage_transformers

* tests for different zarr_formats in test_storage_transformers

* ignore mypy arg-type error for deprecation test

* fix typing in tests

* test_chunk_key_encoding

* test_invalid_v2_arguments

* test_array_repr

* type annotation for parse_array_metadata

* test_v2_and_v3_exist_at_same_path

* remove duplicate check for dimension_separator in v3

* tests for invalid arguments in creation

* format

* revert typing

* document changes

Author: brokkoli71

changes/3049.misc.rst

@@ -0,0 +1 @@
+Added tests for ``AsyncArray``, ``Array`` and removed duplicate argument parsing.

src/zarr/api/asynchronous.py

@@ -1019,11 +1019,6 @@ async def create(
         warnings.warn("object_codec is not yet implemented", RuntimeWarning, stacklevel=2)
     if read_only is not None:
         warnings.warn("read_only is not yet implemented", RuntimeWarning, stacklevel=2)
-    if dimension_separator is not None and zarr_format == 3:
-        raise ValueError(
-            "dimension_separator is not supported for zarr format 3, use chunk_key_encoding instead"
-        )
-
     if order is not None:
         _warn_order_kwarg()
     if write_empty_chunks is not None:

src/zarr/core/array.py

@@ -140,7 +140,8 @@ def parse_array_metadata(data: Any) -> ArrayMetadata:
     if isinstance(data, ArrayMetadata):
         return data
     elif isinstance(data, dict):
-        if data["zarr_format"] == 3:
+        zarr_format = data.get("zarr_format")
+        if zarr_format == 3:
             meta_out = ArrayV3Metadata.from_dict(data)
             if len(meta_out.storage_transformers) > 0:
                 msg = (
@@ -149,9 +150,11 @@ def parse_array_metadata(data: Any) -> ArrayMetadata:
                 )
                 raise ValueError(msg)
             return meta_out
-        elif data["zarr_format"] == 2:
+        elif zarr_format == 2:
             return ArrayV2Metadata.from_dict(data)
-    raise TypeError
+        else:
+            raise ValueError(f"Invalid zarr_format: {zarr_format}. Expected 2 or 3")
+    raise TypeError  # pragma: no cover
 
 
 def create_codec_pipeline(metadata: ArrayMetadata) -> CodecPipeline:
@@ -160,8 +163,7 @@ def create_codec_pipeline(metadata: ArrayMetadata) -> CodecPipeline:
     elif isinstance(metadata, ArrayV2Metadata):
         v2_codec = V2Codec(filters=metadata.filters, compressor=metadata.compressor)
         return get_pipeline_class().from_codecs([v2_codec])
-    else:
-        raise TypeError
+    raise TypeError  # pragma: no cover
 
 
 async def get_array_metadata(
@@ -268,17 +270,6 @@ def __init__(
         store_path: StorePath,
         config: ArrayConfigLike | None = None,
     ) -> None:
-        if isinstance(metadata, dict):
-            zarr_format = metadata["zarr_format"]
-            # TODO: remove this when we extensively type the dict representation of metadata
-            _metadata = cast(dict[str, JSON], metadata)
-            if zarr_format == 2:
-                metadata = ArrayV2Metadata.from_dict(_metadata)
-            elif zarr_format == 3:
-                metadata = ArrayV3Metadata.from_dict(_metadata)
-            else:
-                raise ValueError(f"Invalid zarr_format: {zarr_format}. Expected 2 or 3")
-
         metadata_parsed = parse_array_metadata(metadata)
         config_parsed = parse_array_config(config)
 

tests/test_api.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import re
 from typing import TYPE_CHECKING
 
 import zarr.codecs
@@ -72,13 +73,19 @@ def test_create(memory_store: Store) -> None:
 
 # TODO: parametrize over everything this function takes
 @pytest.mark.parametrize("store", ["memory"], indirect=True)
-def test_create_array(store: Store) -> None:
+def test_create_array(store: Store, zarr_format: ZarrFormat) -> None:
     attrs: dict[str, JSON] = {"foo": 100}  # explicit type annotation to avoid mypy error
     shape = (10, 10)
     path = "foo"
     data_val = 1
     array_w = create_array(
-        store, name=path, shape=shape, attributes=attrs, chunks=shape, dtype="uint8"
+        store,
+        name=path,
+        shape=shape,
+        attributes=attrs,
+        chunks=shape,
+        dtype="uint8",
+        zarr_format=zarr_format,
     )
     array_w[:] = data_val
     assert array_w.shape == shape
@@ -87,18 +94,27 @@ def test_create_array(store: Store) -> None:
 
 
 @pytest.mark.parametrize("write_empty_chunks", [True, False])
-def test_write_empty_chunks_warns(write_empty_chunks: bool) -> None:
+def test_write_empty_chunks_warns(write_empty_chunks: bool, zarr_format: ZarrFormat) -> None:
     """
     Test that using the `write_empty_chunks` kwarg on array access will raise a warning.
     """
     match = "The `write_empty_chunks` keyword argument .*"
     with pytest.warns(RuntimeWarning, match=match):
         _ = zarr.array(
-            data=np.arange(10), shape=(10,), dtype="uint8", write_empty_chunks=write_empty_chunks
+            data=np.arange(10),
+            shape=(10,),
+            dtype="uint8",
+            write_empty_chunks=write_empty_chunks,
+            zarr_format=zarr_format,
         )
 
     with pytest.warns(RuntimeWarning, match=match):
-        _ = zarr.create(shape=(10,), dtype="uint8", write_empty_chunks=write_empty_chunks)
+        _ = zarr.create(
+            shape=(10,),
+            dtype="uint8",
+            write_empty_chunks=write_empty_chunks,
+            zarr_format=zarr_format,
+        )
 
 
 @pytest.mark.parametrize("path", ["foo", "/", "/foo", "///foo/bar"])
@@ -115,18 +131,18 @@ def test_open_normalized_path(
     assert node.path == normalize_path(path)
 
 
-async def test_open_array(memory_store: MemoryStore) -> None:
+async def test_open_array(memory_store: MemoryStore, zarr_format: ZarrFormat) -> None:
     store = memory_store
 
     # open array, create if doesn't exist
-    z = open(store=store, shape=100)
+    z = open(store=store, shape=100, zarr_format=zarr_format)
     assert isinstance(z, Array)
     assert z.shape == (100,)
 
     # open array, overwrite
     # store._store_dict = {}
     store = MemoryStore()
-    z = open(store=store, shape=200)
+    z = open(store=store, shape=200, zarr_format=zarr_format)
     assert isinstance(z, Array)
     assert z.shape == (200,)
 
@@ -140,7 +156,16 @@ async def test_open_array(memory_store: MemoryStore) -> None:
 
     # path not found
     with pytest.raises(FileNotFoundError):
-        open(store="doesnotexist", mode="r")
+        open(store="doesnotexist", mode="r", zarr_format=zarr_format)
+
+
+@pytest.mark.parametrize("store", ["memory", "local", "zip"], indirect=True)
+def test_v2_and_v3_exist_at_same_path(store: Store) -> None:
+    zarr.create_array(store, shape=(10,), dtype="uint8", zarr_format=3)
+    zarr.create_array(store, shape=(10,), dtype="uint8", zarr_format=2)
+    msg = f"Both zarr.json (Zarr format 3) and .zarray (Zarr format 2) metadata objects exist at {store}. Zarr v3 will be used."
+    with pytest.warns(UserWarning, match=re.escape(msg)):
+        zarr.open(store=store, mode="r")
 
 
 @pytest.mark.parametrize("store", ["memory"], indirect=True)
@@ -163,9 +188,9 @@ async def test_open_group(memory_store: MemoryStore) -> None:
     assert "foo" in g
 
     # open group, overwrite
-    # g = open_group(store=store)
-    # assert isinstance(g, Group)
-    # assert "foo" not in g
+    g = open_group(store=store, mode="w")
+    assert isinstance(g, Group)
+    assert "foo" not in g
 
     # open group, read-only
     store_cls = type(store)
@@ -308,7 +333,6 @@ def test_open_with_mode_w_minus(tmp_path: pathlib.Path) -> None:
         zarr.open(store=tmp_path, mode="w-")
 
 
-@pytest.mark.parametrize("zarr_format", [2, 3])
 def test_array_order(zarr_format: ZarrFormat) -> None:
     arr = zarr.ones(shape=(2, 2), order=None, zarr_format=zarr_format)
     expected = zarr.config.get("array.order")
@@ -324,7 +348,6 @@ def test_array_order(zarr_format: ZarrFormat) -> None:
 
 
 @pytest.mark.parametrize("order", ["C", "F"])
-@pytest.mark.parametrize("zarr_format", [2, 3])
 def test_array_order_warns(order: MemoryOrder | None, zarr_format: ZarrFormat) -> None:
     with pytest.warns(RuntimeWarning, match="The `order` keyword argument .*"):
         arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format)
@@ -1095,13 +1118,16 @@ def test_open_falls_back_to_open_group() -> None:
     assert group.attrs == {"key": "value"}
 
 
-async def test_open_falls_back_to_open_group_async() -> None:
+async def test_open_falls_back_to_open_group_async(zarr_format: ZarrFormat) -> None:
     # https://github.com/zarr-developers/zarr-python/issues/2309
     store = MemoryStore()
-    await zarr.api.asynchronous.open_group(store, attributes={"key": "value"})
+    await zarr.api.asynchronous.open_group(
+        store, attributes={"key": "value"}, zarr_format=zarr_format
+    )
 
     group = await zarr.api.asynchronous.open(store=store)
     assert isinstance(group, zarr.core.group.AsyncGroup)
+    assert group.metadata.zarr_format == zarr_format
     assert group.attrs == {"key": "value"}
 
 
@@ -1137,13 +1163,14 @@ async def test_metadata_validation_error() -> None:
     ["local", "memory", "zip"],
     indirect=True,
 )
-def test_open_array_with_mode_r_plus(store: Store) -> None:
+def test_open_array_with_mode_r_plus(store: Store, zarr_format: ZarrFormat) -> None:
     # 'r+' means read/write (must exist)
     with pytest.raises(FileNotFoundError):
-        zarr.open_array(store=store, mode="r+")
-    zarr.ones(store=store, shape=(3, 3))
+        zarr.open_array(store=store, mode="r+", zarr_format=zarr_format)
+    zarr.ones(store=store, shape=(3, 3), zarr_format=zarr_format)
     z2 = zarr.open_array(store=store, mode="r+")
     assert isinstance(z2, Array)
+    assert z2.metadata.zarr_format == zarr_format
     result = z2[:]
     assert isinstance(result, NDArrayLike)
     assert (result == 1).all()