deploy: 173cbdf

Author: DriesSchaumont

.github/workflows/integration-test.yml

@@ -68,7 +68,7 @@ jobs:
 
     - uses: viash-io/viash-actions/setup@v5
 
-    - uses: nf-core/setup-nextflow@v1.5.2
+    - uses: nf-core/setup-nextflow@v2.0.0
 
     # use cache
     - name: Cache resources data

.github/workflows/release-build.yml

@@ -61,7 +61,7 @@ jobs:
 
     - uses: viash-io/viash-actions/setup@v5
 
-    - uses: nf-core/setup-nextflow@v1.5.2
+    - uses: nf-core/setup-nextflow@v2.0.0
 
     # use cache
     - name: Cache resources data

.github/workflows/viash-test.yml

@@ -77,7 +77,7 @@ jobs:
       run: |
         echo "matrix=$(jq -c '[ .[] | 
           { 
-            "name": .functionality.namespace,
+            "name": .functionality.name,
             "namespace_separator": .platforms | map(select(.type == "docker"))[0].namespace_separator,
             "namespace": .functionality.namespace,
             "config": .info.config,

CHANGELOG.md

@@ -1,3 +1,28 @@
+# openpipelines 1.0.0rc2
+
+## BUG FIXES
+
+* Cellranger multi: Fix using a relative input path for `--vdj_inner_enrichment_primers` (PR #717)
+
+* `dataflow/split_modalities`: remove unused `compression` argument. Use `output_compression` instead (PR #714).
+
+* `metadata/grep_annotation_column`: fix calculating fraction when an input observation has no counts, which caused
+the result to be out of bounds.
+
+* Fix `--output` argument not working for several workflows (PR #740).
+
+## MINOR CHANGES
+
+* `metadata/grep_annotation_column`: Added more logging output (PR #697).
+
+* `metadata/add_id` and `metadata/grep_annotation_column`: Bump python to 3.11 (PR #697).
+
+* Bump viash to 0.8.5 (PR #697)
+
+* `dataflow/split_modalities`: add more logging output and bump python to 3.12 (PR #714).
+
+* `correction/cellbender`: Update nextflow resource labels from `singlecpu` and `lowmem` to `midcpu` and `midmem` (PR #736)
+
 # openpipelines 1.0.0rc1
 
 ## BREAKING CHANGES

_viash.yaml

@@ -1,4 +1,4 @@
-viash_version: 0.8.4
+viash_version: 0.8.5
 
 source: src
 target: target
@@ -8,6 +8,7 @@ config_mods: |
   .functionality.requirements.commands := ['ps']
   .functionality.arguments[.multiple == true].multiple_sep := ";"
   .functionality.argument_groups[true].arguments[.multiple == true].multiple_sep := ";"
+  .functionality.test_resources += {path: 'src/base/openpipelinetestutils', dest: 'openpipelinetestutils'}
   .platforms[.type == 'docker'].namespace_separator := '_'
   .platforms[.type == 'docker'].target_registry := 'ghcr.io'
   .platforms[.type == 'docker'].target_organization := 'openpipelines-bio'

src/base/openpipelinetestutils/.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+build
+eggs/
+*.egg
+*.egg-info/

src/base/openpipelinetestutils/__init__.py

@@ -0,0 +1,135 @@
+import mudata
+import anndata
+import pandas as pd
+import numpy as np
+from scipy.sparse import issparse
+from mudata import MuData
+from pathlib import Path
+from pandas.testing import assert_frame_equal
+from typing import Literal
+from .typing import AnnotationObjectOrPathLike
+
+
+def _read_if_needed(anndata_mudata_path_or_obj):
+    if isinstance(anndata_mudata_path_or_obj, (str, Path)):
+        return mudata.read(anndata_mudata_path_or_obj)
+    if isinstance(anndata_mudata_path_or_obj, (mudata.MuData, anndata.AnnData)):
+        return anndata_mudata_path_or_obj.copy()
+    raise AssertionError("Expected 'Path', 'str' to MuData/AnnData "
+                         "file or MuData/AnnData object.")
+
+def _assert_same_annotation_object_class(left, right):
+    assert type(left) == type(right), (f"Two objects are not of the same class:"
+                                       f"\n[Left]:{type(left)}\n[right]:{type(right)}")
+
+
+def assert_mudata_modality_keys_equal(left, right):
+    left_keys = set(left.mod.keys())
+    right_keys = set(right.mod.keys())
+    if left_keys!= right_keys:
+        raise AssertionError("MuData modalities differ:"
+                             f"\n[left]:{left_keys}\n[right]:{right_keys}")
+
+def assert_shape_equal(left: AnnotationObjectOrPathLike, right: AnnotationObjectOrPathLike):
+    left, right = _read_if_needed(left), _read_if_needed(right)
+    _assert_same_annotation_object_class(left, right)
+    if left.shape != right.shape:
+        raise AssertionError(f"{type(left).__name__} shapes differ:"
+                             f"\n[left]:{left.shape}\n[right]:{right.shape}")
+    if isinstance(left, MuData):
+        assert_mudata_modality_keys_equal(left, right)
+        for mod_name, modality in left.mod.items(): 
+            assert_shape_equal(modality, right[mod_name])
+ 
+
+def assert_obs_names_equal(left: AnnotationObjectOrPathLike, right: AnnotationObjectOrPathLike, 
+                           *args, **kwargs):
+    left, right = _read_if_needed(left), _read_if_needed(right)
+    _assert_same_annotation_object_class(left, right)
+    pd.testing.assert_index_equal(left.obs_names, right.obs_names, *args, **kwargs)
+    if isinstance(left, MuData):
+        assert_mudata_modality_keys_equal(left, right)
+        for mod_name, modality in left.mod.items(): 
+            assert_obs_names_equal(modality, right[mod_name])
+
+
+def assert_var_names_equal(left: AnnotationObjectOrPathLike, right: AnnotationObjectOrPathLike, 
+                           *args, **kwargs):
+    left, right = _read_if_needed(left), _read_if_needed(right)
+    _assert_same_annotation_object_class(left, right)
+    pd.testing.assert_index_equal(left.var_names, right.var_names, *args, **kwargs)
+    if isinstance(left, MuData):
+        assert_mudata_modality_keys_equal(left, right)
+        for mod_name, modality in left.mod.items(): 
+            assert_var_names_equal(modality, right[mod_name])
+
+
+def assert_annotation_frame_equal(annotation_attr: Literal["obs", "var"], 
+                                   left: AnnotationObjectOrPathLike, right: AnnotationObjectOrPathLike, 
+                                   sort=False, *args, **kwargs):
+    if not annotation_attr in ("obs", "var"):
+        raise ValueError("annotation_attr should be 'obs', or 'var'")
+    left, right = _read_if_needed(left), _read_if_needed(right)
+    _assert_same_annotation_object_class(left, right)
+    left_frame, right_frame = getattr(left, annotation_attr), getattr(right, annotation_attr)
+    if sort:
+        left_frame, right_frame = left_frame.sort_index(inplace=False), right_frame.sort_index(inplace=False)
+    assert_frame_equal(left_frame, right_frame, *args, **kwargs)
+    if isinstance(left, MuData):
+        assert_mudata_modality_keys_equal(left, right)
+        for mod_name, modality in left.mod.items(): 
+            assert_annotation_frame_equal(annotation_attr, modality, 
+                                           right[mod_name], sort=sort, *args, **kwargs)
+
+def _assert_layer_equal(left, right):
+    if issparse(left):
+        if not issparse(right):
+           raise AssertionError("Layers differ:\n[left]: sparse\n[right]: not sparse")
+        if left.getformat() != right.getformat():
+            raise AssertionError("Layers format differ:"
+                                 f"\n[left]:{left.getformat()}\n[right]: {right.getformat()}")
+        assert np.all(left.indices == right.indices), "Layers differ: indices are not the same"
+        assert np.all(left.indptr == right.indptr), "Layers differ: index pointers are not the same"
+        np.testing.assert_allclose(left.data, right.data, 
+                                  err_msg="Layers data differs.", equal_nan=True)
+    else:
+        if issparse(right):
+            raise AssertionError("Layers differ:\n[left]: not sparse\n[right]: sparse")
+        np.testing.assert_allclose(left, right, 
+                                   err_msg="Layers data differs.", equal_nan=True)
+        
+
+def assert_layers_equal(left: AnnotationObjectOrPathLike,
+                        right: AnnotationObjectOrPathLike):
+    left, right = _read_if_needed(left), _read_if_needed(right)
+    _assert_same_annotation_object_class(left, right)
+    if left.raw is not None:
+       _assert_layer_equal(left.raw, right.raw)
+    else:
+        if right.raw:
+            raise AssertionError("Layer .raw differs: "
+                                 f"\n[left]:{left.raw}\n[right]:{right}")
+    if left.X is not None:
+        _assert_layer_equal(left.X, right.X)
+    if left.layers:
+        assert right.layers and (left.layers.keys() == right.layers.keys()), \
+        "Avaiable layers differ:" \
+        f"\n[left]:{left.layers}\n[right]{right.layers}"
+        for layer_name, layer in left.layers.items():
+            _assert_layer_equal(layer, right.layers[layer_name])
+    if isinstance(left, MuData):
+        assert_mudata_modality_keys_equal(left, right)
+        for mod_name, modality in left.mod.items(): 
+            assert_layers_equal(modality, right[mod_name])
+
+
+def assert_annotation_objects_equal(left: AnnotationObjectOrPathLike,
+                                    right: AnnotationObjectOrPathLike,
+                                    check_data=True):
+    left, right = _read_if_needed(left), _read_if_needed(right)
+    _assert_same_annotation_object_class(left, right)
+    assert_shape_equal(left, right)
+    assert_annotation_frame_equal("obs", left, right)
+    assert_annotation_frame_equal("var", left, right)
+    if check_data:
+        assert_layers_equal(left, right)

src/base/openpipelinetestutils/asserters.py

@@ -0,0 +1,23 @@
+from uuid import uuid4
+import pytest
+
+@pytest.fixture
+def random_path(tmp_path):
+    def wrapper(extension=None):
+        extension = "" if not extension else f".{extension}"
+        return tmp_path / f"{uuid4()}{extension}"
+    return wrapper 
+
+@pytest.fixture
+def random_h5mu_path(random_path):
+    def wrapper():
+        return random_path(extension="h5mu")
+    return wrapper
+
+@pytest.fixture
+def write_mudata_to_file(random_h5mu_path):
+    def wrapper(mudata_obj):
+        output_path = random_h5mu_path()
+        mudata_obj.write(output_path)
+        return output_path
+    return wrapper

src/base/openpipelinetestutils/fixtures.py

@@ -0,0 +1,5 @@
+[build-system]
+requires = [
+  "setuptools >= 40.9.0",
+]
+build-backend = "setuptools.build_meta"