convert pytests to nf-test for concatenation (#1806)

<!--
# nf-core/sarek pull request

Many thanks for contributing to nf-core/sarek!

Please fill in the appropriate checklist below (delete whatever is not
relevant).
These are the most common things requested on pull requests (PRs).

Remember that PRs should be made against the dev branch, unless you're
preparing a pipeline release.

Learn more about contributing:
[CONTRIBUTING.md](https://github.com/nf-core/sarek/tree/master/.github/CONTRIBUTING.md)
-->

## PR checklist

- [ ] This comment contains a description of changes (with reason).
- [ ] If you've fixed a bug or added code that should be tested, add
tests!
- [ ] If you've added a new tool - have you followed the pipeline
conventions in the [contribution
docs](https://github.com/nf-core/sarek/tree/master/.github/CONTRIBUTING.md)
- [ ] If necessary, also make a PR on the nf-core/sarek _branch_ on the
[nf-core/test-datasets](https://github.com/nf-core/test-datasets)
repository.
- [ ] Make sure your code lints (`nf-core pipelines lint`).
- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker
--outdir <OUTDIR>`).
- [ ] Check for unexpected warnings in debug mode (`nextflow run .
-profile debug,test,docker --outdir <OUTDIR>`).
- [ ] Usage Documentation in `docs/usage.md` is updated.
- [ ] Output Documentation in `docs/output.md` is updated.
- [ ] `CHANGELOG.md` is updated.
- [ ] `README.md` is updated (including new tool citations and
authors/contributors).

CHANGELOG.md

@@ -10,13 +10,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - [1803](https://github.com/nf-core/sarek/pull/1803) - Back to dev
+- [1806](https://github.com/nf-core/sarek/pull/1806) - Use `nft-vcf` for nf-test vcf assertions
 
 ### Changed
 
+- [1806](https://github.com/nf-core/sarek/pull/1806) - Migrate pipeline pytest vcf concatenation tests to nf-test
+
 ### Fixed
 
+- [1806](https://github.com/nf-core/sarek/pull/1806) - Fix some nf-test assertions
+
 ### Removed
 
+- [1806](https://github.com/nf-core/sarek/pull/1806) - Remove some files publication with the --concatenate_vcfs options
+
 ### Dependencies
 
 | Dependency | Old version | New version |

conf/modules/post_variant_calling.config

@@ -19,10 +19,12 @@ process {
     withName: 'GERMLINE_VCFS_CONCAT'{
         ext.args   = { "-a" }
         ext.when   = { params.concatenate_vcfs }
-        publishDir = [
-            //specify to avoid publishing, overwritten otherwise
-            enabled: false
-        ]
+        publishDir = [ enabled: false ]
+    }
+
+    withName: 'ADD_INFO_TO_VCF'{
+        ext.when   = { params.concatenate_vcfs }
+        publishDir = [ enabled: false ]
     }
 
     withName: 'GERMLINE_VCFS_CONCAT_SORT'{
@@ -37,6 +39,7 @@ process {
     withName: 'TABIX_EXT_VCF' {
         ext.prefix = { "${input.baseName}" }
         ext.when   = { params.concatenate_vcfs }
+        publishDir = [ enabled: false ]
     }
 
     withName: 'TABIX_GERMLINE_VCFS_CONCAT_SORT'{

nf-test.config

@@ -16,5 +16,6 @@ config {
     plugins {
         load "[email protected]"
         load "[email protected]"
+        load "[email protected]"
     }
 }

subworkflows/local/vcf_concatenate_germline/main.nf

@@ -3,14 +3,13 @@
 //
 
 // Concatenation of germline vcf-files
-include { ADD_INFO_TO_VCF                                     } from '../../../modules/local/add_info_to_vcf/main'
-include { TABIX_BGZIPTABIX as TABIX_EXT_VCF                   } from '../../../modules/nf-core/tabix/bgziptabix/main'
-include { BCFTOOLS_CONCAT  as GERMLINE_VCFS_CONCAT            } from '../../../modules/nf-core/bcftools/concat/main'
-include { BCFTOOLS_SORT    as GERMLINE_VCFS_CONCAT_SORT       } from '../../../modules/nf-core/bcftools/sort/main'
-include { TABIX_TABIX      as TABIX_GERMLINE_VCFS_CONCAT_SORT } from '../../../modules/nf-core/tabix/tabix/main'
+include { ADD_INFO_TO_VCF                                } from '../../../modules/local/add_info_to_vcf'
+include { BCFTOOLS_CONCAT as GERMLINE_VCFS_CONCAT        } from '../../../modules/nf-core/bcftools/concat'
+include { BCFTOOLS_SORT as GERMLINE_VCFS_CONCAT_SORT     } from '../../../modules/nf-core/bcftools/sort'
+include { TABIX_BGZIPTABIX as TABIX_EXT_VCF              } from '../../../modules/nf-core/tabix/bgziptabix'
+include { TABIX_TABIX as TABIX_GERMLINE_VCFS_CONCAT_SORT } from '../../../modules/nf-core/tabix/tabix'
 
 workflow CONCATENATE_GERMLINE_VCFS {
-
     take:
     vcfs
 
@@ -22,7 +21,7 @@ workflow CONCATENATE_GERMLINE_VCFS {
     TABIX_EXT_VCF(ADD_INFO_TO_VCF.out.vcf)
 
     // Gather vcfs and vcf-tbis for concatenating germline-vcfs
-    germline_vcfs_with_tbis = TABIX_EXT_VCF.out.gz_tbi.map{ meta, vcf, tbi -> [ meta.subMap('id'), vcf, tbi ] }.groupTuple()
+    germline_vcfs_with_tbis = TABIX_EXT_VCF.out.gz_tbi.map { meta, vcf, tbi -> [meta.subMap('id'), vcf, tbi] }.groupTuple()
 
     GERMLINE_VCFS_CONCAT(germline_vcfs_with_tbis)
     GERMLINE_VCFS_CONCAT_SORT(GERMLINE_VCFS_CONCAT.out.vcf)
@@ -36,7 +35,6 @@ workflow CONCATENATE_GERMLINE_VCFS {
     versions = versions.mix(TABIX_GERMLINE_VCFS_CONCAT_SORT.out.versions)
 
     emit:
-    vcfs = germline_vcfs_with_tbis // post processed vcfs
-
+    vcfs     = germline_vcfs_with_tbis // post processed vcfs
     versions // channel: [ versions.yml ]
 }

tests/alignment_from_everything.nf.test

@@ -23,9 +23,8 @@ nextflow_pipeline {
             def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}'])
             // stable_path: All files in ${params.outdir}/ with stable content
             def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
-            // cram_files: All cram files
-            def cram_files  = getAllFilesFromDir(params.outdir, include: ['**/*.cram'])
-            def fasta       = params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta'
+            // vcf_files: All vcf files
+            def vcf_files  = getAllFilesFromDir(params.outdir, include: ['**/*.vcf.gz'])
             assertAll(
                 { assert workflow.success},
                 { assert snapshot(
@@ -37,8 +36,8 @@ nextflow_pipeline {
                     stable_name,
                     // All files with stable contents
                     stable_path,
-                    // All cram files
-                    cram_files.collect{ file -> [ file.getName(), cram(file.toString(), fasta).getReadsMD5() ] }
+                    // All vcf files
+                    vcf_files.collect{ file -> [ file.getName(), path(file.toString()).vcf.variantsMD5] }
                 ).match() }
             )
         }

tests/alignment_from_everything.nf.test.snap

@@ -323,13 +323,20 @@
                 "test.strelka.variants.TsTv.count:md5,fa27f678965b7cba6a92efcd039f802a"
             ],
             [
-
+                [
+                    "test.strelka.genome.vcf.gz",
+                    "d23db2df1603e0949f7788a3c3640833"
+                ],
+                [
+                    "test.strelka.variants.vcf.gz",
+                    "48ca98a5c4b715b71241fea2ae740179"
+                ]
             ]
         ],
         "meta": {
-            "nf-test": "0.9.0",
-            "nextflow": "24.04.4"
+            "nf-test": "0.9.2",
+            "nextflow": "24.10.4"
         },
-        "timestamp": "2024-11-09T21:51:21.998984"
+        "timestamp": "2025-02-24T19:01:12.937596048"
     }
-}
+}

tests/alignment_to_fastq.nf.test

@@ -6,7 +6,7 @@ nextflow_pipeline {
     tag "pipeline_sarek"
     profile "alignment_to_fastq"
 
-    test("Run with profile test,alignment_to_fastq") {
+    test("Run with profile test,alignment_to_fastq | --save_mapped --save_output_as_bam") {
 
         when {
             params {
@@ -23,9 +23,7 @@ nextflow_pipeline {
             def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}'])
             // stable_path: All files in ${params.outdir}/ with stable content
             def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
-            // cram_files: All cram files
-            def cram_files  = getAllFilesFromDir(params.outdir, include: ['**/*.cram'])
-            def fasta       = params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta'
+            def vcf_files  = getAllFilesFromDir(params.outdir, include: ['**/*.vcf.gz'])
             assertAll(
                 { assert workflow.success},
                 { assert snapshot(
@@ -37,8 +35,8 @@ nextflow_pipeline {
                     stable_name,
                     // All files with stable contents
                     stable_path,
-                    // All cram files
-                    cram_files.collect{ file -> [ file.getName(), cram(file.toString(), fasta).getReadsMD5() ] }
+                    // All vcf files
+                    vcf_files.collect{ file -> [ file.getName(), path(file.toString()).vcf.variantsMD5] }
                 ).match() }
             )
         }

tests/alignment_to_fastq.nf.test.snap

@@ -1,5 +1,5 @@
 {
-    "Run with profile test,alignment_to_fastq": {
+    "Run with profile test,alignment_to_fastq | --save_mapped --save_output_as_bam": {
         "content": [
             27,
             {
@@ -323,13 +323,20 @@
                 "test.strelka.variants.TsTv.count:md5,fa27f678965b7cba6a92efcd039f802a"
             ],
             [
-
+                [
+                    "test.strelka.genome.vcf.gz",
+                    "d23db2df1603e0949f7788a3c3640833"
+                ],
+                [
+                    "test.strelka.variants.vcf.gz",
+                    "48ca98a5c4b715b71241fea2ae740179"
+                ]
             ]
         ],
         "meta": {
-            "nf-test": "0.9.0",
-            "nextflow": "24.04.4"
+            "nf-test": "0.9.2",
+            "nextflow": "24.10.4"
         },
-        "timestamp": "2024-11-09T17:49:40.35031"
+        "timestamp": "2025-02-24T19:04:06.023379801"
     }
-}
+}

tests/annotation_bcfann.nf.test

@@ -22,6 +22,8 @@ nextflow_pipeline {
             def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}'])
             // stable_path: All files in ${params.outdir}/ with stable content
             def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
+            // vcf_files: All vcf files
+            def vcf_files  = getAllFilesFromDir(params.outdir, include: ['**/*.vcf.gz'])
             assertAll(
                 { assert workflow.success},
                 { assert snapshot(
@@ -32,7 +34,9 @@ nextflow_pipeline {
                     // All stable path name, with a relative path
                     stable_name,
                     // All files with stable contents
-                    stable_path
+                    stable_path,
+                    // All vcf files
+                    vcf_files.collect{ file -> [ file.getName(), path(file.toString()).vcf.variantsMD5] }
                 ).match() }
             )
         }

tests/annotation_bcfann.nf.test.snap

@@ -29,12 +29,18 @@
             ],
             [
                 "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f"
+            ],
+            [
+                [
+                    "test_BCF.ann.vcf.gz",
+                    "bc7bf3ee9e8430e064c539eb81e59bf9"
+                ]
             ]
         ],
         "meta": {
-            "nf-test": "0.9.1",
-            "nextflow": "24.10.0"
+            "nf-test": "0.9.2",
+            "nextflow": "24.10.4"
         },
-        "timestamp": "2024-10-29T09:38:54.991004"
+        "timestamp": "2025-02-24T17:11:13.075740065"
     }
-}
+}

tests/annotation_merge.nf.test

@@ -24,6 +24,8 @@ nextflow_pipeline {
             def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}'])
             // stable_path: All files in ${params.outdir}/ with stable content
             def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
+            // vcf_files: All vcf files
+            def vcf_files  = getAllFilesFromDir(params.outdir, include: ['**/*.vcf.gz'])
             assertAll(
                 { assert workflow.success},
                 { assert snapshot(
@@ -34,7 +36,9 @@ nextflow_pipeline {
                     // All stable path name, with a relative path
                     stable_name,
                     // All files with stable contents
-                    stable_path
+                    stable_path,
+                    // All vcf files
+                    vcf_files.collect{ file -> [ file.getName(), path(file.toString()).vcf.variantsMD5] }
                 ).match() }
             )
         }
@@ -59,6 +63,8 @@ nextflow_pipeline {
             def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}'])
             // stable_path: All files in ${params.outdir}/ with stable content
             def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
+            // vcf_files: All vcf files
+            def vcf_files  = getAllFilesFromDir(params.outdir, include: ['**/*.vcf.gz'])
             assertAll(
                 { assert workflow.success},
                 { assert snapshot(
@@ -69,7 +75,9 @@ nextflow_pipeline {
                     // All stable path name, with a relative path
                     stable_name,
                     // All files with stable contents
-                    stable_path
+                    stable_path,
+                    // All vcf files
+                    vcf_files.collect{ file -> [ file.getName(), path(file.toString()).vcf.variantsMD5] }
                 ).match() }
             )
         }

tests/annotation_merge.nf.test.snap

@@ -97,13 +97,27 @@
                 "vep-general-stats.txt:md5,71c994ae4221384f4e22459723d29cd0",
                 "vep.txt:md5,20570f3e4e51407b860a31d7e1d59de0",
                 "test_snpEff.genes.txt:md5,130536bf0237d7f3f746d32aaa32840a"
+            ],
+            [
+                [
+                    "test_VEP.ann.vcf.gz",
+                    "d41d8cd98f00b204e9800998ecf8427e"
+                ],
+                [
+                    "test_snpEff.ann.vcf.gz",
+                    "fd266e623365c7d1837bc9eb088fe70c"
+                ],
+                [
+                    "test_snpEff_VEP.ann.vcf.gz",
+                    "d41d8cd98f00b204e9800998ecf8427e"
+                ]
             ]
         ],
         "meta": {
-            "nf-test": "0.9.1",
-            "nextflow": "24.10.0"
+            "nf-test": "0.9.2",
+            "nextflow": "24.10.4"
         },
-        "timestamp": "2024-10-29T10:03:16.229715"
+        "timestamp": "2025-02-24T17:16:02.176798697"
     },
     "Run with profile test | --tools merge": {
         "content": [
@@ -194,12 +208,18 @@
                 "snpeff_variant_effects_region.txt:md5,05efd324edadced17ba3cd2b7714af57",
                 "vep-general-stats.txt:md5,57563be109a57f6edfa427b2b2c310ba",
                 "vep.txt:md5,bf54f689bb0ccab5e1566e48373f768c"
+            ],
+            [
+                [
+                    "test_snpEff_VEP.ann.vcf.gz",
+                    "d41d8cd98f00b204e9800998ecf8427e"
+                ]
             ]
         ],
         "meta": {
-            "nf-test": "0.9.1",
-            "nextflow": "24.10.0"
+            "nf-test": "0.9.2",
+            "nextflow": "24.10.4"
         },
-        "timestamp": "2024-10-29T09:56:59.385257"
+        "timestamp": "2025-02-24T17:14:45.665437291"
     }
-}
+}

tests/annotation_snpeff.nf.test

@@ -23,6 +23,8 @@ nextflow_pipeline {
             def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}'])
             // stable_path: All files in ${params.outdir}/ with stable content
             def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
+            // vcf_files: All vcf files
+            def vcf_files  = getAllFilesFromDir(params.outdir, include: ['**/*.vcf.gz'])
             assertAll(
                 { assert workflow.success},
                 { assert snapshot(
@@ -33,7 +35,9 @@ nextflow_pipeline {
                     // All stable path name, with a relative path
                     stable_name,
                     // All files with stable contents
-                    stable_path
+                    stable_path,
+                    // All vcf files
+                    vcf_files.collect{ file -> [ file.getName(), path(file.toString()).vcf.variantsMD5] }
                 ).match() }
             )
         }

tests/annotation_snpeff.nf.test.snap

@@ -97,12 +97,18 @@
                 "snpeff_qualities.txt:md5,4c059b4e8bf0a64940ad1d6e30efd3a6",
                 "snpeff_variant_effects_region.txt:md5,05efd324edadced17ba3cd2b7714af57",
                 "test_snpEff.genes.txt:md5,130536bf0237d7f3f746d32aaa32840a"
+            ],
+            [
+                [
+                    "test_snpEff.ann.vcf.gz",
+                    "fd266e623365c7d1837bc9eb088fe70c"
+                ]
             ]
         ],
         "meta": {
-            "nf-test": "0.9.1",
-            "nextflow": "24.10.0"
+            "nf-test": "0.9.2",
+            "nextflow": "24.10.4"
         },
-        "timestamp": "2024-10-29T10:16:00.665699"
+        "timestamp": "2025-02-24T17:17:34.571501001"
     }
-}
+}