Backport PR #690 to 0.12.x (#691)

Author: DriesSchaumont

CHANGELOG.md

@@ -1,3 +1,9 @@
+# openpipelines 0.12.6
+
+## BUG FIXES
+
+* `move_obsm_to_obs`: fix setting output columns when they already exist (PR #690).
+
 # openpipelines 0.12.5
 
 ## BUG FIXES

src/metadata/move_obsm_to_obs/script.py

@@ -47,9 +47,11 @@ def setup_logger():
 try:
     logger.info(f".obs names: {mod_data.obs_names}")
     logger.info(f".obsm index: {obsm_matrix.index}")
-    mod_data.obs = mod_data.obs.merge(obsm_matrix, how="left",
-                                      validate="one_to_one",
-                                      left_index=True, right_index=True)
+    new_obs = mod_data.obs.drop(obsm_matrix.columns, axis=1, errors="ignore")
+    new_obs = new_obs.merge(obsm_matrix, how="left",
+                            validate="one_to_one",
+                            left_index=True, right_index=True)
+    mod_data.obs = new_obs
 except MergeError as e:
     raise ValueError(f"Could not join .obsm matrix at {par['obsm_key']} to .obs because there "
                      "are some observation that are not overlapping between the two matrices "

src/metadata/move_obsm_to_obs/test.py

@@ -1,6 +1,7 @@
 import pytest
 import re
 import pandas as pd
+import uuid
 from anndata import AnnData
 from mudata import MuData, read_h5mu
 from subprocess import CalledProcessError
@@ -10,7 +11,7 @@
     'functionality_name': 'move_obsm_to_obs',
     'resources_dir': 'resources_test/',
     'executable': 'target/docker/metadata/move_obsm_to_obs/move_obsm_to_obs',
-    'config': '/home/di/code/openpipeline/src/metadata/move_obsm_to_obs/config.vsh.yaml'
+    'config': 'src/metadata/move_obsm_to_obs/config.vsh.yaml'
 }
 ## VIASH END
 
@@ -31,9 +32,9 @@ def h5mu():
 @pytest.fixture
 def write_temp_h5mu(tmp_path):
     def wrapper(test_h5mu): 
-        test_h5mu_path = tmp_path / "input.h5mu"
-        test_h5mu.write_h5mu(test_h5mu_path.name)
-        return test_h5mu_path.name
+        test_h5mu_path = tmp_path / f"{str(uuid.uuid4())}.h5mu"
+        test_h5mu.write_h5mu(test_h5mu_path)
+        return test_h5mu_path
     return wrapper
 
 @pytest.fixture
@@ -80,6 +81,33 @@ def test_error_non_existing_modality(run_component, h5mu, write_temp_h5mu, tmp_p
                     ])
     re.search(r"ValueError: Modality foo does not exist\.",
         err.value.stdout.decode('utf-8'))
+    
+def test_execute_twice_overwrites(run_component, h5mu, write_temp_h5mu, tmp_path):
+    output_run_1 = tmp_path/ "output1.h5mu"
+    run_component(["--input", write_temp_h5mu(h5mu),
+                   "--modality", "mod1",
+                   "--obsm_key", "obsm_key",
+                   "--output", output_run_1
+                   ])
+    output_data_run_1 = read_h5mu(output_run_1)
+    output_data_run_1.mod["mod1"].obsm = \
+        {"obsm_key": pd.DataFrame([["dolor", "amet"], ["jommeke", "filiberke"]],
+                                  index=output_data_run_1.mod["mod1"].obs_names,
+                                  columns=["obsm_col1", "obsm_col2"])}
+    
+    output_run_2 = tmp_path / "output2.h5mu"
+    input_run_2 = write_temp_h5mu(output_data_run_1)
+    run_component(["--input", input_run_2,
+                   "--modality", "mod1",
+                   "--obsm_key", "obsm_key",
+                   "--output", output_run_2
+                   ])
+    assert output_run_2.is_file(), "Some output file must have been created."
+    output_data = read_h5mu(output_run_2)
+    pd.testing.assert_index_equal(output_data.mod['mod1'].obs.index, pd.Index(['obs1', 'obs2']))
+    pd.testing.assert_index_equal(output_data.mod['mod1'].obs.columns, 
+                                  pd.Index(['Obs', 'sample_id', 'obsm_key_obsm_col1', 'obsm_key_obsm_col2']))
+    assert 'obsm_key' not in output_data.mod['mod1'].obsm
 
 if __name__ == '__main__':
     exit(pytest.main([__file__]))

target/docker/annotate/popv/.config.vsh.yaml

@@ -1,7 +1,7 @@
 functionality:
   name: "popv"
   namespace: "annotate"
-  version: "0.12.5"
+  version: "0.12.6"
   authors:
   - name: "Matthias Beyens"
     roles:
@@ -341,6 +341,6 @@ info:
   output: "/home/runner/work/openpipeline/openpipeline/target/docker/annotate/popv"
   executable: "/home/runner/work/openpipeline/openpipeline/target/docker/annotate/popv/popv"
   viash_version: "0.7.5"
-  git_commit: "5fa6062c47eb4206e52bc17f2cbcc04c27c4e953"
+  git_commit: "8bf9c29a2c1223c2e4139d0372743537798fd3ea"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
-  git_tag: "0.12.4-3-g5fa6062c47"
+  git_tag: "0.12.5-3-g8bf9c29a2c"

target/docker/annotate/popv/popv

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# popv 0.12.5
+# popv 0.12.6
 # 
 # This wrapper script is auto-generated by viash 0.7.5 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
@@ -159,7 +159,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 # ViashHelp: Display helpful explanation about this executable
 function ViashHelp {
-  echo "popv 0.12.5"
+  echo "popv 0.12.6"
   echo ""
   echo "Performs popular major vote cell typing on single cell sequence data using"
   echo "multiple algorithms. Note that this is a one-shot version of PopV."
@@ -488,10 +488,10 @@ RUN cd /opt && git clone --depth 1 https://github.com/YosefLab/PopV.git && \
 
 LABEL org.opencontainers.image.authors="Matthias Beyens, Robrecht Cannoodt"
 LABEL org.opencontainers.image.description="Companion container for running component annotate popv"
-LABEL org.opencontainers.image.created="2024-02-02T13:20:49Z"
+LABEL org.opencontainers.image.created="2024-02-03T20:49:49Z"
 LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline"
-LABEL org.opencontainers.image.revision="5fa6062c47eb4206e52bc17f2cbcc04c27c4e953"
-LABEL org.opencontainers.image.version="0.12.5"
+LABEL org.opencontainers.image.revision="8bf9c29a2c1223c2e4139d0372743537798fd3ea"
+LABEL org.opencontainers.image.version="0.12.6"
 
 VIASHDOCKER
 }
@@ -642,7 +642,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "popv 0.12.5"
+            echo "popv 0.12.6"
             exit
             ;;
         --input)

target/docker/cluster/leiden/.config.vsh.yaml

@@ -1,7 +1,7 @@
 functionality:
   name: "leiden"
   namespace: "cluster"
-  version: "0.12.5"
+  version: "0.12.6"
   authors:
   - name: "Dries De Maeyer"
     roles:
@@ -214,6 +214,6 @@ info:
   output: "/home/runner/work/openpipeline/openpipeline/target/docker/cluster/leiden"
   executable: "/home/runner/work/openpipeline/openpipeline/target/docker/cluster/leiden/leiden"
   viash_version: "0.7.5"
-  git_commit: "5fa6062c47eb4206e52bc17f2cbcc04c27c4e953"
+  git_commit: "8bf9c29a2c1223c2e4139d0372743537798fd3ea"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
-  git_tag: "0.12.4-3-g5fa6062c47"
+  git_tag: "0.12.5-3-g8bf9c29a2c"

target/docker/cluster/leiden/leiden

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# leiden 0.12.5
+# leiden 0.12.6
 # 
 # This wrapper script is auto-generated by viash 0.7.5 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
@@ -158,7 +158,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 # ViashHelp: Display helpful explanation about this executable
 function ViashHelp {
-  echo "leiden 0.12.5"
+  echo "leiden 0.12.6"
   echo ""
   echo "Cluster cells using the Leiden algorithm [Traag18] implemented in the Scanpy"
   echo "framework [Wolf18]."
@@ -445,10 +445,10 @@ RUN pip install --upgrade pip && \
 
 LABEL org.opencontainers.image.authors="Dries De Maeyer"
 LABEL org.opencontainers.image.description="Companion container for running component cluster leiden"
-LABEL org.opencontainers.image.created="2024-02-02T13:20:49Z"
+LABEL org.opencontainers.image.created="2024-02-03T20:49:48Z"
 LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline"
-LABEL org.opencontainers.image.revision="5fa6062c47eb4206e52bc17f2cbcc04c27c4e953"
-LABEL org.opencontainers.image.version="0.12.5"
+LABEL org.opencontainers.image.revision="8bf9c29a2c1223c2e4139d0372743537798fd3ea"
+LABEL org.opencontainers.image.version="0.12.6"
 
 VIASHDOCKER
 }
@@ -599,7 +599,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "leiden 0.12.5"
+            echo "leiden 0.12.6"
             exit
             ;;
         --input)

target/docker/compression/compress_h5mu/.config.vsh.yaml

@@ -1,7 +1,7 @@
 functionality:
   name: "compress_h5mu"
   namespace: "compression"
-  version: "0.12.5"
+  version: "0.12.6"
   authors:
   - name: "Dries Schaumont"
     roles:
@@ -162,6 +162,6 @@ info:
   output: "/home/runner/work/openpipeline/openpipeline/target/docker/compression/compress_h5mu"
   executable: "/home/runner/work/openpipeline/openpipeline/target/docker/compression/compress_h5mu/compress_h5mu"
   viash_version: "0.7.5"
-  git_commit: "5fa6062c47eb4206e52bc17f2cbcc04c27c4e953"
+  git_commit: "8bf9c29a2c1223c2e4139d0372743537798fd3ea"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
-  git_tag: "0.12.4-3-g5fa6062c47"
+  git_tag: "0.12.5-3-g8bf9c29a2c"

target/docker/compression/compress_h5mu/compress_h5mu

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# compress_h5mu 0.12.5
+# compress_h5mu 0.12.6
 # 
 # This wrapper script is auto-generated by viash 0.7.5 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
@@ -158,7 +158,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 # ViashHelp: Display helpful explanation about this executable
 function ViashHelp {
-  echo "compress_h5mu 0.12.5"
+  echo "compress_h5mu 0.12.6"
   echo ""
   echo "Compress a MuData file."
   echo ""
@@ -408,10 +408,10 @@ RUN pip install --upgrade pip && \
 
 LABEL org.opencontainers.image.authors="Dries Schaumont"
 LABEL org.opencontainers.image.description="Companion container for running component compression compress_h5mu"
-LABEL org.opencontainers.image.created="2024-02-02T13:20:47Z"
+LABEL org.opencontainers.image.created="2024-02-03T20:49:49Z"
 LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline"
-LABEL org.opencontainers.image.revision="5fa6062c47eb4206e52bc17f2cbcc04c27c4e953"
-LABEL org.opencontainers.image.version="0.12.5"
+LABEL org.opencontainers.image.revision="8bf9c29a2c1223c2e4139d0372743537798fd3ea"
+LABEL org.opencontainers.image.version="0.12.6"
 
 VIASHDOCKER
 }
@@ -562,7 +562,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "compress_h5mu 0.12.5"
+            echo "compress_h5mu 0.12.6"
             exit
             ;;
         --input)

target/docker/compression/tar_extract/.config.vsh.yaml

@@ -1,7 +1,7 @@
 functionality:
   name: "tar_extract"
   namespace: "compression"
-  version: "0.12.5"
+  version: "0.12.6"
   arguments:
   - type: "file"
     name: "--input"
@@ -101,6 +101,6 @@ info:
   output: "/home/runner/work/openpipeline/openpipeline/target/docker/compression/tar_extract"
   executable: "/home/runner/work/openpipeline/openpipeline/target/docker/compression/tar_extract/tar_extract"
   viash_version: "0.7.5"
-  git_commit: "5fa6062c47eb4206e52bc17f2cbcc04c27c4e953"
+  git_commit: "8bf9c29a2c1223c2e4139d0372743537798fd3ea"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
-  git_tag: "0.12.4-3-g5fa6062c47"
+  git_tag: "0.12.5-3-g8bf9c29a2c"