Backport #913 to 2.0.x (#917)

Author: DriesSchaumont

CHANGELOG.md

@@ -1,3 +1,9 @@
+# openpipelines 2.0.0-rc.2
+
+## MINOR CHANGES
+
+* `dataflow/split_h5mu`: Optimize resource usage of the component (PR #917, backported from PR #913).
+
 # openpipelines 2.0.0-rc.1
 
 ## BREAKING CHANGES

src/dataflow/split_h5mu/script.py

@@ -1,6 +1,7 @@
 import mudata as mu
 import pandas as pd
 import re
+import gc
 from pathlib import Path
 from collections import defaultdict
 
@@ -100,6 +101,11 @@ def main():
         mdata_obs.mod[par["modality"]] = adata_obs
         mdata_obs.write_h5mu(output_dir / mdata_obs_name, compression=par["output_compression"])
 
+        # avoid keeping files in memory
+        del mdata_obs
+        del adata_obs
+        gc.collect()
+
     logger.info(f"Writing output_files CSV file to {par['output_files']}")
     df = pd.DataFrame({"name": obs_features_s, "filename": obs_files})
     df.to_csv(par["output_files"], index=False)

target/executable/annotate/celltypist/.config.vsh.yaml

@@ -1,6 +1,6 @@
 name: "celltypist"
 namespace: "annotate"
-version: "2.0.0-rc.1"
+version: "2.0.0-rc.2"
 authors:
 - name: "Jakub Majercik"
   roles:
@@ -394,9 +394,9 @@ build_info:
   output: "target/executable/annotate/celltypist"
   executable: "target/executable/annotate/celltypist/celltypist"
   viash_version: "0.9.0"
-  git_commit: "f060d75d1633b6f25a83be2c483b393e08662673"
+  git_commit: "686217bf7f7dadccb6d8a730b953d8124da01c58"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
-  git_tag: "1.0.0-7-gf060d75d16"
+  git_tag: "2.0.0-rc.1-1-g686217bf7f"
 package_config:
   name: "openpipeline"
   version: "dev"
@@ -413,7 +413,7 @@ package_config:
     .resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
     .runners[.type == 'nextflow'].directives.tag := '$id'\n.runners[.type == 'nextflow'].config.script\
     \ := 'includeConfig(\"nextflow_labels.config\")'"
-  - ".version := \"2.0.0-rc.1\""
+  - ".version := \"2.0.0-rc.2\""
   - ".engines[.type == 'docker'].target_tag := '2.0.0-rc.1'"
   organization: "openpipelines-bio"
   links:

target/executable/annotate/celltypist/celltypist

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# celltypist 2.0.0-rc.1
+# celltypist 2.0.0-rc.2
 # 
 # This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
@@ -175,7 +175,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 # ViashHelp: Display helpful explanation about this executable
 function ViashHelp {
-  echo "celltypist 2.0.0-rc.1"
+  echo "celltypist 2.0.0-rc.2"
   echo ""
   echo "Automated cell type annotation tool for scRNA-seq datasets on the basis of"
   echo "logistic regression classifiers optimised by the stochastic gradient descent"
@@ -596,10 +596,10 @@ RUN pip install --upgrade pip && \
 
 LABEL org.opencontainers.image.authors="Jakub Majercik, Weiwei Schultz"
 LABEL org.opencontainers.image.description="Companion container for running component annotate celltypist"
-LABEL org.opencontainers.image.created="2024-11-25T05:49:53Z"
+LABEL org.opencontainers.image.created="2024-11-27T14:19:07Z"
 LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline"
-LABEL org.opencontainers.image.revision="f060d75d1633b6f25a83be2c483b393e08662673"
-LABEL org.opencontainers.image.version="2.0.0-rc.1"
+LABEL org.opencontainers.image.revision="686217bf7f7dadccb6d8a730b953d8124da01c58"
+LABEL org.opencontainers.image.version="2.0.0-rc.2"
 
 VIASHDOCKER
   fi
@@ -735,7 +735,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "celltypist 2.0.0-rc.1"
+            echo "celltypist 2.0.0-rc.2"
             exit
             ;;
         --input)

target/executable/annotate/onclass/.config.vsh.yaml

@@ -1,6 +1,6 @@
 name: "onclass"
 namespace: "annotate"
-version: "2.0.0-rc.1"
+version: "2.0.0-rc.2"
 authors:
 - name: "Jakub Majercik"
   roles:
@@ -336,9 +336,9 @@ build_info:
   output: "target/executable/annotate/onclass"
   executable: "target/executable/annotate/onclass/onclass"
   viash_version: "0.9.0"
-  git_commit: "f060d75d1633b6f25a83be2c483b393e08662673"
+  git_commit: "686217bf7f7dadccb6d8a730b953d8124da01c58"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
-  git_tag: "1.0.0-7-gf060d75d16"
+  git_tag: "2.0.0-rc.1-1-g686217bf7f"
 package_config:
   name: "openpipeline"
   version: "dev"
@@ -355,7 +355,7 @@ package_config:
     .resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
     .runners[.type == 'nextflow'].directives.tag := '$id'\n.runners[.type == 'nextflow'].config.script\
     \ := 'includeConfig(\"nextflow_labels.config\")'"
-  - ".version := \"2.0.0-rc.1\""
+  - ".version := \"2.0.0-rc.2\""
   - ".engines[.type == 'docker'].target_tag := '2.0.0-rc.1'"
   organization: "openpipelines-bio"
   links:

target/executable/annotate/onclass/onclass

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# onclass 2.0.0-rc.1
+# onclass 2.0.0-rc.2
 # 
 # This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
@@ -174,7 +174,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 # ViashHelp: Display helpful explanation about this executable
 function ViashHelp {
-  echo "onclass 2.0.0-rc.1"
+  echo "onclass 2.0.0-rc.2"
   echo ""
   echo "OnClass is a python package for single-cell cell type annotation. It uses the"
   echo "Cell Ontology to capture the cell type similarity."
@@ -561,10 +561,10 @@ RUN pip install --upgrade pip && \
 
 LABEL org.opencontainers.image.authors="Jakub Majercik"
 LABEL org.opencontainers.image.description="Companion container for running component annotate onclass"
-LABEL org.opencontainers.image.created="2024-11-25T05:49:52Z"
+LABEL org.opencontainers.image.created="2024-11-27T14:19:06Z"
 LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline"
-LABEL org.opencontainers.image.revision="f060d75d1633b6f25a83be2c483b393e08662673"
-LABEL org.opencontainers.image.version="2.0.0-rc.1"
+LABEL org.opencontainers.image.revision="686217bf7f7dadccb6d8a730b953d8124da01c58"
+LABEL org.opencontainers.image.version="2.0.0-rc.2"
 
 VIASHDOCKER
   fi
@@ -700,7 +700,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "onclass 2.0.0-rc.1"
+            echo "onclass 2.0.0-rc.2"
             exit
             ;;
         --input)

target/executable/annotate/popv/.config.vsh.yaml

@@ -1,6 +1,6 @@
 name: "popv"
 namespace: "annotate"
-version: "2.0.0-rc.1"
+version: "2.0.0-rc.2"
 authors:
 - name: "Matthias Beyens"
   roles:
@@ -351,9 +351,9 @@ build_info:
   output: "target/executable/annotate/popv"
   executable: "target/executable/annotate/popv/popv"
   viash_version: "0.9.0"
-  git_commit: "f060d75d1633b6f25a83be2c483b393e08662673"
+  git_commit: "686217bf7f7dadccb6d8a730b953d8124da01c58"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
-  git_tag: "1.0.0-7-gf060d75d16"
+  git_tag: "2.0.0-rc.1-1-g686217bf7f"
 package_config:
   name: "openpipeline"
   version: "dev"
@@ -370,7 +370,7 @@ package_config:
     .resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
     .runners[.type == 'nextflow'].directives.tag := '$id'\n.runners[.type == 'nextflow'].config.script\
     \ := 'includeConfig(\"nextflow_labels.config\")'"
-  - ".version := \"2.0.0-rc.1\""
+  - ".version := \"2.0.0-rc.2\""
   - ".engines[.type == 'docker'].target_tag := '2.0.0-rc.1'"
   organization: "openpipelines-bio"
   links:

target/executable/annotate/popv/popv

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# popv 2.0.0-rc.1
+# popv 2.0.0-rc.2
 # 
 # This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
@@ -175,7 +175,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 # ViashHelp: Display helpful explanation about this executable
 function ViashHelp {
-  echo "popv 2.0.0-rc.1"
+  echo "popv 2.0.0-rc.2"
   echo ""
   echo "Performs popular major vote cell typing on single cell sequence data using"
   echo "multiple algorithms. Note that this is a one-shot version of PopV."
@@ -561,10 +561,10 @@ RUN cd /opt && git clone --depth 1 https://github.com/YosefLab/PopV.git
 
 LABEL org.opencontainers.image.authors="Matthias Beyens, Robrecht Cannoodt"
 LABEL org.opencontainers.image.description="Companion container for running component annotate popv"
-LABEL org.opencontainers.image.created="2024-11-25T05:49:52Z"
+LABEL org.opencontainers.image.created="2024-11-27T14:19:06Z"
 LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline"
-LABEL org.opencontainers.image.revision="f060d75d1633b6f25a83be2c483b393e08662673"
-LABEL org.opencontainers.image.version="2.0.0-rc.1"
+LABEL org.opencontainers.image.revision="686217bf7f7dadccb6d8a730b953d8124da01c58"
+LABEL org.opencontainers.image.version="2.0.0-rc.2"
 
 VIASHDOCKER
   fi
@@ -700,7 +700,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "popv 2.0.0-rc.1"
+            echo "popv 2.0.0-rc.2"
             exit
             ;;
         --input)

target/executable/annotate/random_forest_annotation/.config.vsh.yaml

@@ -1,6 +1,6 @@
 name: "random_forest_annotation"
 namespace: "annotate"
-version: "2.0.0-rc.1"
+version: "2.0.0-rc.2"
 authors:
 - name: "Jakub Majercik"
   roles:
@@ -355,9 +355,9 @@ build_info:
   output: "target/executable/annotate/random_forest_annotation"
   executable: "target/executable/annotate/random_forest_annotation/random_forest_annotation"
   viash_version: "0.9.0"
-  git_commit: "f060d75d1633b6f25a83be2c483b393e08662673"
+  git_commit: "686217bf7f7dadccb6d8a730b953d8124da01c58"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
-  git_tag: "1.0.0-7-gf060d75d16"
+  git_tag: "2.0.0-rc.1-1-g686217bf7f"
 package_config:
   name: "openpipeline"
   version: "dev"
@@ -374,7 +374,7 @@ package_config:
     .resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
     .runners[.type == 'nextflow'].directives.tag := '$id'\n.runners[.type == 'nextflow'].config.script\
     \ := 'includeConfig(\"nextflow_labels.config\")'"
-  - ".version := \"2.0.0-rc.1\""
+  - ".version := \"2.0.0-rc.2\""
   - ".engines[.type == 'docker'].target_tag := '2.0.0-rc.1'"
   organization: "openpipelines-bio"
   links:

target/executable/annotate/random_forest_annotation/random_forest_annotation

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# random_forest_annotation 2.0.0-rc.1
+# random_forest_annotation 2.0.0-rc.2
 # 
 # This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
@@ -174,7 +174,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 # ViashHelp: Display helpful explanation about this executable
 function ViashHelp {
-  echo "random_forest_annotation 2.0.0-rc.1"
+  echo "random_forest_annotation 2.0.0-rc.2"
   echo ""
   echo "Automated cell type annotation tool for scRNA-seq datasets on the basis of"
   echo "random forest."
@@ -576,10 +576,10 @@ RUN pip install --upgrade pip && \
 
 LABEL org.opencontainers.image.authors="Jakub Majercik"
 LABEL org.opencontainers.image.description="Companion container for running component annotate random_forest_annotation"
-LABEL org.opencontainers.image.created="2024-11-25T05:49:51Z"
+LABEL org.opencontainers.image.created="2024-11-27T14:19:05Z"
 LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline"
-LABEL org.opencontainers.image.revision="f060d75d1633b6f25a83be2c483b393e08662673"
-LABEL org.opencontainers.image.version="2.0.0-rc.1"
+LABEL org.opencontainers.image.revision="686217bf7f7dadccb6d8a730b953d8124da01c58"
+LABEL org.opencontainers.image.version="2.0.0-rc.2"
 
 VIASHDOCKER
   fi
@@ -715,7 +715,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "random_forest_annotation 2.0.0-rc.1"
+            echo "random_forest_annotation 2.0.0-rc.2"
             exit
             ;;
         --input)