ENH: Major update to DSL2 and nf-neuro

.devcontainer/Dockerfile

@@ -0,0 +1,94 @@
+FROM ubuntu:24.04
+
+USER root
+
+RUN apt-get update --quiet && \
+    export DEBIAN_FRONTEND=noninteractive && \
+    apt-get install --quiet --yes \
+        apt-transport-https \
+        apt-utils \
+        sudo \
+        git \
+        less \
+        wget \
+        curl \
+        tree \
+        graphviz \
+        vim \
+        software-properties-common && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install Apptainer (Singularity)
+RUN add-apt-repository -y ppa:apptainer/ppa && \
+    apt-get update --quiet && \
+    apt install -y apptainer && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set up directories
+RUN mkdir -p /workspaces/.nextflow
+
+# Install Seqera Platform "Tower Agent"
+RUN curl -fSL https://github.com/seqeralabs/tower-agent/releases/latest/download/tw-agent-linux-x86_64 > tw-agent && \
+    chmod +x tw-agent && \
+    mv tw-agent /usr/local/bin/tw-agent
+
+# Uncomment if we need to pin the Nextflow version
+ENV NXF_EDGE=0
+ENV NXF_VER=24.10.4
+ENV NXF_HOME=/workspaces/.nextflow
+
+ENV NFNEURO_RAW_REPOSITORY="https://raw.githubusercontent.com/scilus/nf-neuro/main"
+ENV NFCORE_MODULES_GIT_REMOTE=https://github.com/scilus/nf-neuro.git
+ENV NFCORE_MODULES_BRANCH=main
+ENV NFCORE_SUBWORKFLOWS_GIT_REMOTE=https://github.com/scilus/nf-neuro.git
+ENV NFCORE_SUBWORKFLOWS_BRANCH=main
+
+# Install conda
+ENV PATH="/root/miniconda3/bin:${PATH}"
+RUN arch=$(uname -m) && \
+    if [ "$arch" = "x86_64" ]; then \
+    MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-py312_25.3.1-1-Linux-x86_64.sh"; \
+    elif [ "$arch" = "aarch64" ]; then \
+    MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-py312_25.3.1-1-Linux-aarch64.sh"; \
+    else \
+    echo "Unsupported architecture: $arch"; \
+    exit 1; \
+    fi && \
+    wget $MINICONDA_URL -O miniconda.sh && \
+    mkdir -p /root/.conda && \
+    bash miniconda.sh -b -p /root/miniconda3 && \
+    rm -f miniconda.sh
+
+# Install nextflow, nf-core, Mamba, and pytest-workflow
+RUN conda install --name=base conda-anaconda-tos && \
+    conda config --add channels bioconda && \
+    conda config --add channels conda-forge && \
+    conda config --set channel_priority strict && \
+    conda tos accept && \
+    conda update --quiet --yes --all && \
+    conda install --quiet --yes --name base \
+        mamba \
+        nextflow=$NXF_VER \
+        nf-core=2.14.1 \
+        nf-test=0.9.2 \
+        black \
+        prettier \
+        pre-commit \
+        linkify-it-py \
+        pytest-workflow && \
+    conda clean --all --force-pkgs-dirs --yes
+
+# Cleanup
+RUN printf 'unset JAVA_TOOL_OPTIONS\n' >> $HOME/.bashrc
+
+# Update Nextflow
+RUN nextflow self-update && nextflow -version
+
+RUN unset JAVA_TOOL_OPTIONS
+ENV PS1='\[\e[3;36m\]${PWD/*//} ->\[\e[0m\] '
+
+# Get history
+RUN SNIPPET="export PROMPT_COMMAND='history -a' && export HISTFILE=/commandhistory/.bash_history" \
+    && echo "$SNIPPET" >> "/root/.bashrc"

.devcontainer/devcontainer.json

@@ -0,0 +1,50 @@
+{
+    "name": "nextflow-training",
+    "build": { "dockerfile": "Dockerfile", "context": ".." },
+    "forwardPorts": [3000],
+    "features": {
+        "ghcr.io/devcontainers/features/docker-in-docker:2": {
+            "dockerDashComposeVersion": "none",
+            "installDockerComposeSwitch": false
+        }
+    },
+    "postCreateCommand": {
+        "identify": "if [ -z \"$CODESPACES\" ]; then echo \"Devcontainers Development\"; else echo \"Codespaces Development\";  fi",
+        "trust": "git config --global --add safe.directory /workspaces"
+    },
+    "mounts": [
+        {
+            "target": "/tmp",
+            "type": "volume"
+        },
+        {
+            "target": "/root/.vscode-server",
+            "type": "volume"
+        },
+        {
+            "target": "/commandhistory",
+            "type": "volume",
+            "source": "nf-neuro-tutorial-${devcontainerId}-bashhistory"
+        }
+    ],    
+    // Configure tool-specific properties.
+    "customizations": {
+        // Configure properties specific to VS Code.
+        "vscode": {
+            // Set *default* container specific settings.json values on container create.
+            "settings": {
+                "python.defaultInterpreterPath": "/opt/conda/bin/python"
+            },
+            // Add the IDs of extensions you want installed when the container is created.
+            "extensions": [
+                "ms-python.python",
+                "ms-python.vscode-pylance",
+                "nf-core.nf-core-extensionpack", // nf-core recommended extensions
+                "nextflow.nextflow", // Nextflow VS Code extension
+                "codezombiech.gitignore", // Language support for .gitignore files
+                "cssho.vscode-svgviewer", // SVG viewer
+                "nf-neuro.nf-neuro-extensionpack"
+            ]
+        }
+    }
+}

.gitattributes

@@ -0,0 +1,3 @@
+* text=auto eol=lf
+*.{cmd,[cC][mM][dD]} text eol=crlf
+*.{bat,[bB][aA][tT]} text eol=crlf

.gitignore

@@ -0,0 +1,9 @@
+.nextflow.log*
+.vscode
+.nextflow
+.nf-test.log
+
+tests/.runs/
+work/
+.nf-test/
+extractor_templates/

.nf-core.yml

@@ -0,0 +1,2 @@
+---
+repository_type: pipeline

README.md

@@ -1,41 +1,118 @@
+# Extractor Flow: streamline filtering pipeline
 
-[![GitHub release (latest by date)](https://img.shields.io/github/v/release/scilus/extractor_flow)](https://github.com/scilus/extractor_flow/releases)
-[![Documentation Status](https://readthedocs.org/projects/extractor_flow/badge/?version=latest)](https://extractor_flow.readthedocs.io/en/latest/?badge=latest)
-[![Docker container badge](https://img.shields.io/docker/v/scilus/scilus-flows?label=docker&logo=docker&logoColor=white)](https://hub.docker.com/r/scilus/scilus-flows)
+ExtractorFlow is a streamline filtering pipeline written in nextflow. In essence, you provide any tractogram file (i.e. `*.trk`) as an input and this pipeline will output which streamlines are deemed as anatomically **plausible** and **implausible** as separate `.trk` files.
 
+Additionnally, as described below, you can use this pipeline to extract recognized (i.e. plausible) bundles of streamlines.
 
-ExtractorFlow pipeline
-======================
+When using this work, please cite using the following:
 
-Usage
------
+```
+PETIT, Laurent, ALI, Kariem Mahdy, RHEAULT, François, et al. The structural connectivity of the human angular gyrus as revealed by microdissection and diffusion tractography. Brain Structure and Function, 2023, vol. 228, no 1, p. 103-120.
+```
 
-See *USAGE* or run `nextflow run main.nf --help`
+## Requirements
+- [Nextflow](https://www.nextflow.io/docs/latest/install.html)
+- [Docker](https://www.docker.com/get-started/) (recommended) or [Apptainer](https://apptainer.org/docs/admin/main/installation.html) depending on the runtime you choose.
 
-Singularity
------------
+You should be able to run the following commands without any errors being printed:
+```bash
+# 1. Make sure Nextflow is installed.
+nextflow -v
 
-If you are on Linux, we recommend using the Singularity container to run ExtractorFlow
+# 2a. Make sure Docker is installed.
+docker ps
 
-Run the command (your working directory has to be the "containers" directory):
+# 2b. Make sure Apptainer is installed.
+apptainer version
+```
 
-`sudo singularity build singularity_extractorflow.sif singularity_extractorflow.def`
+## Getting started
+###  Understand the input
+This nextflow pipeline has **two** mandatory arguments that any user has to provide in order to run the pipeline.
+1. `--input` 
+2. `--templates_dir`
 
-Then you can start the pipeline using this command line:
+Both of these arguments point to two different directories that have their own particular structure:
 
+#### `--input`
+This argument points to the directory holding all the tractograms you wish to filter. Also, within this directory, each tractogram should probably be associated with a T1w image (in diffusion space). This T1w image is used to register your tractograms to MNI space, which is the space where the filtering is performed. **If no T1w image is provided, your tractograms are assumed to be already in the appropriate template (MNI) space**. This said, your directory structure should look like the following:
 ```
-nextflow main.nf --root=/path_to_your_data/ -with-singularity singularity_extractorflow.sif -resume
+input_example
+├── subject-01
+│   ├── subject-01__t1.nii.gz
+│   └── subject-01_tracking.trk
+├── subject-02
+│   ├── subject-02__t1.nii.gz
+│   └── subject-02_tracking.trk
+├── ...
+└── subject-n
+    ├── subject-n__t1.nii.gz
+    └── subject-n_tracking.trk
 ```
+If your tractograms are already in the right space, keep the same structure, but omit the `*t1.nii.gz` files.
+
+#### `--templates_dir`  
+To simplify the usage of this pipeline and to avoid cluttering the containers used, the user must download extract the different templates used and the lists that are used during filtering in this pipeline (don't worry, it's a simple process).
 
+All you have to do is execute the following commands to download and extract the needed files into a new directory that we called `extractor_templates`.
 
-Docker
-------
-If you are on MacOS or Windows, we recommend using the Docker container to run ExtractorFlow.
+```bash
+wget https://github.com/scilus/extractor_flow/raw/refs/heads/master/containers/filtering_lists.tar.bz2 && wget https://github.com/scilus/extractor_flow/raw/refs/heads/master/containers/templates_and_ROIs.tar.bz2
+
+mkdir -p extractor_templates
+
+tar -xjf filtering_lists.tar.bz2 -C extractor_templates && rm filtering_lists.tar.bz2
+tar -xjf templates_and_ROIs.tar.bz2 -C extractor_templates && rm templates_and_ROIs.tar.bz2
+```
 
-You can build docker image using this command (your working directory has to be the "containers" directory):
+> If you don't have the `wget` and/or the `tar` commands and you don't want (or can't) to install them, you can always download the two archives manually from [here](https://github.com/scilus/extractor_flow/raw/refs/heads/master/containers/filtering_lists.tar.bz2) and [here](https://github.com/scilus/extractor_flow/raw/refs/heads/master/containers/templates_and_ROIs.tar.bz2) and extract them into a new directory called `extractor_templates`. Just make sure that you have the following directory structure.
 
-`docker build -t extractor_flow .`
+From this point, you should have a directory containing the following structure that you'll use to provide as a value to the `--template_dir` argument in the following steps:
+```
+extractor_templates/
+├── filtering_lists
+│   ├── ...
+└── templates_and_ROIs
+    └── ...
+```
+
+#### Complete arguments list
+To get a complete list of the available arguments you can provide, always refer to the usage printed by the nextflow script as follows:
+```bash
+nextflow run scilus/extractor_flow --help
+```
+
+### Standard filtering using Docker (recommended).
+The following example gives a general idea on what arguments you can provide. The key part is to select the `docker` profile as well as providing the mandatory arguments (i.e. `--input` & `--templates_dir`):
+```bash
+nextflow run scilus/extractor_flow \  
+    --input <input_folder> \  
+    --templates_dir <path/to/extractor_templates> \ 
+    -profile docker \  
+    [--orig] 
+    [--keep_intermediate_steps]
+    [-resume] 
+```
 
+### Standard filering using Apptainer/Singularity.  
+To use apptainer containers, you should only have to swap the profile used to `apptainer` as shown in the following example:
+```bash
+nextflow run scilus/extractor_flow \  
+    --input <input_folder> \  
+    --templates_dir <path/to/extractor_templates> \ 
+    -profile apptainer \ 
+    [--orig] 
+    [--keep_intermediate_steps]
+    [-resume] 
 ```
-nextflow main.nf --root=/path_to_your_data/ -with-docker extractor_flow:latest -resume
+### Filtering and bundle extraction.  
+Notice, in the following example, the addition of the `extract_bundles` profile. This will trigger a few additionnal processes that will separate and organize new files refering to the bundles identified by this flow. The example is using Docker (as the docker profile is enabled), but the same applies for Apptainer.
+```bash
+nextflow run scilus/extractor_flow \  
+    --input <input_folder> \  
+    --templates_dir <path/to/extractor_templates> \ 
+    -profile docker,extract_bundles \ 
+    [--orig]  
+    [--keep_intermediate_steps]  
+    [-resume]
 ```

USAGE

@@ -20,6 +20,16 @@ DESCRIPTION
                                         └── S2
                                             └── *.trk
                                             └── *_t1.nii.gz (diff space, optional)
+    --templates_dir=/path/to/templates_dir
+                                        ABSOLUTE PATH to directory containing the templates
+                                        and the filtering lists (e.g. JHU atlas). Structure
+                                        should look like:
+                                        [templates_dir]
+                                        ├── templates_and_ROIs
+                                        │   ├── JHU*.nii.gz
+                                        └── filtering_lists
+                                            └── filtering_list_*.txt
+
 
 OPTIONAL ARGUMENTS (current value)
 
@@ -33,13 +43,13 @@ OPTIONAL ARGUMENTS (current value)
 
     --quick_registration                If set, will choose antsRegistrationSyNQuick.sh instead of antsRegistrationSyN.sh ($quick_registration).
 
-    --processes_bet_register_t1         Number of processes for T1 brain extraction task ($processes_bet_register_t1).
     --processes_major_filtering         Number of processes for the major filtering task ($processes_major_filtering).
     --processes                         The number of parallel processes to launch ($cpu_count).
                                         Only affects the local scheduler.
 
 AVAILABLE PROFILES (using -profile option (e.g. -profile macos,extract_bundles))
 
-macos                                   When this profile is used, ExtractorFlow will modify a parameter (scratch) for MacOS users.
+docker                                  When this profile is used, all processes will execute within Docker containers (recommended).
+apptainer                               When this profile is used, all processes will execute within Apptainer containers.
 fully_reproducible                      When this profile is used, all the parameters will be set to have 100% reproducible results.
-extract_bundles                         Extract bundles
+extract_bundles                         The flow will perform additional processing to extract all recognized bundles.

config/resources.config

@@ -0,0 +1,18 @@
+
+
+params.max_cpus = 4
+params.max_time = 24.h
+
+
+profiles {
+    default_github_runner {
+        params.max_memory = 5.GB
+    }
+    big_mem_github_runner {
+        params.max_memory = 16.GB
+    }
+    devcontainer {
+        params.max_cpus = query_container_limits('cpus')
+        params.max_memory = query_container_limits('memory')
+    }
+}