Backport PR #678 to 0.10.x (#684)

* log1p: fix input layer argument being unused. (#678)

* log1p: fix input layer argument being unused.

* Add CHANGELOG entry

* Remove target folder

* Update CI

* deploy: e7a3192

* Revert "Update CI"

This reverts commit e7a3192.

---------

Co-authored-by: DriesSchaumont <[email protected]>

Author: DriesSchaumont

CHANGELOG.md

@@ -1,3 +1,9 @@
+# openpipelines 0.10.2
+
+## BUG FIXES
+
+* `transform/log1p`: fix `--input_layer` argument not functionning (PR #678). 
+
 # openpipelines 0.10.1
 
 ## MINOR CHANGES

src/transform/log1p/run_test.py

@@ -1,36 +1,58 @@
 from os import path
 import mudata as mu
 import numpy as np
+import scanpy as sc
+import pandas as pd
 import sys
 import pytest
 import sys
+import uuid
 from operator import attrgetter
 
 ## VIASH START
 meta = {
     'functionality_name': 'lognorm',
     'resources_dir': 'resources_test/',
-    'config': '/home/di/code/openpipeline/src/transform/log1p/config.vsh.yaml',
+    'config': './src/transform/log1p/config.vsh.yaml',
     'executable': "../../target/docker/transform/log1p/log1p"
 }
 
 
 ## VIASH END
 
 @pytest.fixture
-def input_path():
-    return f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"
+def input_data():
+    return mu.read_h5mu(f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu").copy()
+
+@pytest.fixture
+def random_h5mu_path(tmp_path):
+    def wrapper():
+        unique_filename = f"{str(uuid.uuid4())}.h5mu"
+        temp_file = tmp_path / unique_filename
+        return temp_file
+    return wrapper
 
 @pytest.mark.parametrize("output_layer", [None, "log_normalized"])
-def test_1logp(run_component, input_path, output_layer):
-    output = "output.h5mu"
+@pytest.mark.parametrize("input_layer", [None, "normalized"])
+def test_1logp(run_component, input_data, output_layer, input_layer, random_h5mu_path):
+    output = random_h5mu_path()
+    if input_layer:
+        mod = input_data.mod["rna"]
+        mod.layers[input_layer] = mod.X.copy()
+        # Overwrite the original layer to make sure
+        # it is not accidentally used as input layer.
+        mod.X[:] = 0
+    input_path = random_h5mu_path()
+    input_data.write(input_path)
     run_args = [
         "--input", input_path,
         "--output", output,
         "--output_compresion", "gzip"
     ]
     if output_layer:
         run_args.extend(["--output_layer", output_layer])
+    if input_layer:
+        run_args.extend(["--input_layer", input_layer])
     run_component(run_args)
     get_output_layer = attrgetter("X") if not output_layer else lambda x: getattr(x, 'layers')[output_layer]
 
@@ -49,16 +71,31 @@ def test_1logp(run_component, input_path, output_layer):
 
     assert rna_in.shape == rna_out.shape, "Should have same shape as before"
     assert prot_in.shape == prot_out.shape, "Should have same shape as before"
+    input_layer_data = rna_in.X if not input_layer else rna_in.layers[input_layer]
+    assert np.mean(input_layer_data) != np.mean(get_output_layer(rna_out)), "Expression should have changed"
 
-    assert np.mean(rna_in.X) != np.mean(get_output_layer(rna_out)), "Expression should have changed"
-
-    nz_row, nz_col = rna_in.X.nonzero()
-    row_corr = np.corrcoef(rna_in.X[nz_row[0],:].toarray().flatten(), get_output_layer(rna_out)[nz_row[0],:].toarray().flatten())[0,1]
-    col_corr = np.corrcoef(rna_in.X[:,nz_col[0]].toarray().flatten(), get_output_layer(rna_out)[:,nz_col[0]].toarray().flatten())[0,1]
+    nz_row, nz_col = input_layer_data.nonzero()
+    row_corr = np.corrcoef(input_layer_data[nz_row[0],:].toarray().flatten(), 
+                           get_output_layer(rna_out)[nz_row[0],:].toarray().flatten())[0,1]
+    col_corr = np.corrcoef(input_layer_data[:,nz_col[0]].toarray().flatten(), 
+                           get_output_layer(rna_out)[:,nz_col[0]].toarray().flatten())[0,1]
     assert row_corr > .1
     assert col_corr > .1
 
     assert 'log1p' in rna_out.uns
 
+    # Make sure that the original input layer has not been overwritten
+    layers_to_test = [None] + list(rna_in.layers.keys())
+    for layer in layers_to_test:
+        if layer != output_layer:
+            in_data = sc.get.var_df(rna_in,
+                                    keys=rna_in.obs_names.to_list(),
+                                    layer=layer)
+            out_data =  sc.get.var_df(rna_out,
+                                      keys=rna_in.obs_names.to_list(),
+                                      layer=layer)
+            pd.testing.assert_frame_equal(in_data, out_data)
+    
+
 if __name__ == '__main__':
     sys.exit(pytest.main([__file__]))

src/transform/log1p/script.py

@@ -1,5 +1,6 @@
 import scanpy as sc
 import mudata as mu
+import anndata as ad
 import sys
 
 ## VIASH START
@@ -39,12 +40,24 @@ def setup_logger():
 mod = par["modality"]
 logger.info("Performing log transformation on modality %s", mod)
 data = mdata.mod[mod]
-new_layer = sc.pp.log1p(data,
-                        base=par["base"],
-                        copy=True if par['output_layer'] else False)
-if new_layer:
-    data.layers[par['output_layer']] = new_layer.X
-    data.uns['log1p'] = new_layer.uns['log1p']
+
+# Make our own copy with not a lot of data
+# this avoid excessive memory usage and accidental overwrites 
+input_layer = data.layers[par["input_layer"]] \
+              if par["input_layer"] else data.X
+data_for_scanpy = ad.AnnData(X=input_layer.copy())
+sc.pp.log1p(data_for_scanpy,
+            base=par["base"],
+            layer=None, # use X
+            copy=False) # allow overwrites in the copy that was made
+
+# Scanpy will overwrite the input layer.
+# So fetch input layer from the copy and use it to populate the output slot
+if par["output_layer"]:
+    data.layers[par["output_layer"]] = data_for_scanpy.X
+else:
+    data.X = data_for_scanpy.X
+data.uns['log1p'] = data_for_scanpy.uns['log1p'].copy()
 
 logger.info("Writing to file %s", par["output"])
 mdata.write_h5mu(filename=par["output"], compression=par["output_compression"])

target/docker/annotate/popv/.config.vsh.yaml

@@ -1,7 +1,7 @@
 functionality:
   name: "popv"
   namespace: "annotate"
-  version: "0.10.1"
+  version: "0.10.2"
   authors:
   - name: "Matthias Beyens"
     roles:
@@ -247,6 +247,7 @@ platforms:
   image: "python:3.9-slim"
   target_organization: "openpipelines-bio"
   target_registry: "ghcr.io"
+  target_tag: "0.10.1"
   namespace_separator: "_"
   resolve_volume: "Automatic"
   chown: true
@@ -343,5 +344,6 @@ info:
   output: "/home/runner/work/openpipeline/openpipeline/target/docker/annotate/popv"
   executable: "/home/runner/work/openpipeline/openpipeline/target/docker/annotate/popv/popv"
   viash_version: "0.7.5"
-  git_commit: "3d2a0f21b49b7c8bc40e44e668191f0a8cb784b5"
+  git_commit: "e7a31922173ed91fb3a087194dd899bae2698824"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
+  git_tag: "0.10.1-3-ge7a3192217"

target/docker/annotate/popv/popv

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# popv 0.10.1
+# popv 0.10.2
 # 
 # This wrapper script is auto-generated by viash 0.7.5 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
@@ -159,7 +159,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 # ViashHelp: Display helpful explanation about this executable
 function ViashHelp {
-  echo "popv 0.10.1"
+  echo "popv 0.10.2"
   echo ""
   echo "Performs popular major vote cell typing on single cell sequence data using"
   echo "multiple algorithms. Note that this is a one-shot version of PopV."
@@ -491,10 +491,10 @@ RUN pip install --upgrade pip && \
 
 LABEL org.opencontainers.image.authors="Matthias Beyens, Robrecht Cannoodt"
 LABEL org.opencontainers.image.description="Companion container for running component annotate popv"
-LABEL org.opencontainers.image.created="2023-09-08T15:50:22Z"
+LABEL org.opencontainers.image.created="2024-01-31T12:04:47Z"
 LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline"
-LABEL org.opencontainers.image.revision="3d2a0f21b49b7c8bc40e44e668191f0a8cb784b5"
-LABEL org.opencontainers.image.version="0.10.1"
+LABEL org.opencontainers.image.revision="e7a31922173ed91fb3a087194dd899bae2698824"
+LABEL org.opencontainers.image.version="0.10.2"
 
 VIASHDOCKER
 }
@@ -645,7 +645,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "popv 0.10.1"
+            echo "popv 0.10.2"
             exit
             ;;
         --input)

target/docker/cluster/leiden/.config.vsh.yaml

@@ -1,7 +1,7 @@
 functionality:
   name: "leiden"
   namespace: "cluster"
-  version: "0.10.1"
+  version: "0.10.2"
   authors:
   - name: "Dries De Maeyer"
     roles:
@@ -140,6 +140,7 @@ platforms:
   image: "python:3.8-slim"
   target_organization: "openpipelines-bio"
   target_registry: "ghcr.io"
+  target_tag: "0.10.1"
   namespace_separator: "_"
   resolve_volume: "Automatic"
   chown: true
@@ -213,5 +214,6 @@ info:
   output: "/home/runner/work/openpipeline/openpipeline/target/docker/cluster/leiden"
   executable: "/home/runner/work/openpipeline/openpipeline/target/docker/cluster/leiden/leiden"
   viash_version: "0.7.5"
-  git_commit: "3d2a0f21b49b7c8bc40e44e668191f0a8cb784b5"
+  git_commit: "e7a31922173ed91fb3a087194dd899bae2698824"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
+  git_tag: "0.10.1-3-ge7a3192217"

target/docker/cluster/leiden/leiden

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# leiden 0.10.1
+# leiden 0.10.2
 # 
 # This wrapper script is auto-generated by viash 0.7.5 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
@@ -158,7 +158,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 # ViashHelp: Display helpful explanation about this executable
 function ViashHelp {
-  echo "leiden 0.10.1"
+  echo "leiden 0.10.2"
   echo ""
   echo "Cluster cells using the Leiden algorithm [Traag18] implemented in the Scanpy"
   echo "framework [Wolf18]."
@@ -445,10 +445,10 @@ RUN pip install --upgrade pip && \
 
 LABEL org.opencontainers.image.authors="Dries De Maeyer"
 LABEL org.opencontainers.image.description="Companion container for running component cluster leiden"
-LABEL org.opencontainers.image.created="2023-09-08T15:50:25Z"
+LABEL org.opencontainers.image.created="2024-01-31T12:04:45Z"
 LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline"
-LABEL org.opencontainers.image.revision="3d2a0f21b49b7c8bc40e44e668191f0a8cb784b5"
-LABEL org.opencontainers.image.version="0.10.1"
+LABEL org.opencontainers.image.revision="e7a31922173ed91fb3a087194dd899bae2698824"
+LABEL org.opencontainers.image.version="0.10.2"
 
 VIASHDOCKER
 }
@@ -599,7 +599,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "leiden 0.10.1"
+            echo "leiden 0.10.2"
             exit
             ;;
         --input)

target/docker/compression/compress_h5mu/.config.vsh.yaml

@@ -1,7 +1,7 @@
 functionality:
   name: "compress_h5mu"
   namespace: "compression"
-  version: "0.10.1"
+  version: "0.10.2"
   authors:
   - name: "Dries Schaumont"
     roles:
@@ -83,6 +83,7 @@ platforms:
   image: "python:3.10-slim"
   target_organization: "openpipelines-bio"
   target_registry: "ghcr.io"
+  target_tag: "0.10.1"
   namespace_separator: "_"
   resolve_volume: "Automatic"
   chown: true
@@ -161,5 +162,6 @@ info:
   output: "/home/runner/work/openpipeline/openpipeline/target/docker/compression/compress_h5mu"
   executable: "/home/runner/work/openpipeline/openpipeline/target/docker/compression/compress_h5mu/compress_h5mu"
   viash_version: "0.7.5"
-  git_commit: "3d2a0f21b49b7c8bc40e44e668191f0a8cb784b5"
+  git_commit: "e7a31922173ed91fb3a087194dd899bae2698824"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
+  git_tag: "0.10.1-3-ge7a3192217"

target/docker/compression/compress_h5mu/compress_h5mu

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# compress_h5mu 0.10.1
+# compress_h5mu 0.10.2
 # 
 # This wrapper script is auto-generated by viash 0.7.5 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
@@ -158,7 +158,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 # ViashHelp: Display helpful explanation about this executable
 function ViashHelp {
-  echo "compress_h5mu 0.10.1"
+  echo "compress_h5mu 0.10.2"
   echo ""
   echo "Compress a MuData file."
   echo ""
@@ -408,10 +408,10 @@ RUN pip install --upgrade pip && \
 
 LABEL org.opencontainers.image.authors="Dries Schaumont"
 LABEL org.opencontainers.image.description="Companion container for running component compression compress_h5mu"
-LABEL org.opencontainers.image.created="2023-09-08T15:50:23Z"
+LABEL org.opencontainers.image.created="2024-01-31T12:04:46Z"
 LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline"
-LABEL org.opencontainers.image.revision="3d2a0f21b49b7c8bc40e44e668191f0a8cb784b5"
-LABEL org.opencontainers.image.version="0.10.1"
+LABEL org.opencontainers.image.revision="e7a31922173ed91fb3a087194dd899bae2698824"
+LABEL org.opencontainers.image.version="0.10.2"
 
 VIASHDOCKER
 }
@@ -562,7 +562,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "compress_h5mu 0.10.1"
+            echo "compress_h5mu 0.10.2"
             exit
             ;;
         --input)

target/docker/compression/tar_extract/.config.vsh.yaml

@@ -1,7 +1,7 @@
 functionality:
   name: "tar_extract"
   namespace: "compression"
-  version: "0.10.1"
+  version: "0.10.2"
   arguments:
   - type: "file"
     name: "--input"
@@ -87,6 +87,7 @@ platforms:
   image: "ubuntu:latest"
   target_organization: "openpipelines-bio"
   target_registry: "ghcr.io"
+  target_tag: "0.10.1"
   namespace_separator: "_"
   resolve_volume: "Automatic"
   chown: true
@@ -100,5 +101,6 @@ info:
   output: "/home/runner/work/openpipeline/openpipeline/target/docker/compression/tar_extract"
   executable: "/home/runner/work/openpipeline/openpipeline/target/docker/compression/tar_extract/tar_extract"
   viash_version: "0.7.5"
-  git_commit: "3d2a0f21b49b7c8bc40e44e668191f0a8cb784b5"
+  git_commit: "e7a31922173ed91fb3a087194dd899bae2698824"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
+  git_tag: "0.10.1-3-ge7a3192217"