From e70d08069485a6dc3d6370b35e13ea4d7be505c7 Mon Sep 17 00:00:00 2001 From: SPearce Date: Mon, 14 Nov 2022 09:27:06 +0000 Subject: [PATCH 1/8] Fix version string spurious text in singularity tests --- modules/nf-core/bbmap/align/main.nf | 2 +- modules/nf-core/bbmap/bbduk/main.nf | 2 +- modules/nf-core/bbmap/bbsplit/main.nf | 4 ++-- modules/nf-core/bbmap/clumpify/main.nf | 2 +- modules/nf-core/bbmap/index/main.nf | 2 +- modules/nf-core/bbmap/pileup/main.nf | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/nf-core/bbmap/align/main.nf b/modules/nf-core/bbmap/align/main.nf index aa1fbe1a31c..2f64b3ce88b 100644 --- a/modules/nf-core/bbmap/align/main.nf +++ b/modules/nf-core/bbmap/align/main.nf @@ -50,7 +50,7 @@ process BBMAP_ALIGN { cat <<-END_VERSIONS > versions.yml "${task.process}": - bbmap: \$(bbversion.sh) + bbmap: \$(bbversion.sh | grep -v "Duplicate cpuset") samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) END_VERSIONS diff --git a/modules/nf-core/bbmap/bbduk/main.nf b/modules/nf-core/bbmap/bbduk/main.nf index 0ae005e4065..ded2fbfa669 100644 --- a/modules/nf-core/bbmap/bbduk/main.nf +++ b/modules/nf-core/bbmap/bbduk/main.nf @@ -37,7 +37,7 @@ process BBMAP_BBDUK { &> ${prefix}.bbduk.log cat <<-END_VERSIONS > versions.yml "${task.process}": - bbmap: \$(bbversion.sh) + bbmap: \$(bbversion.sh | grep -v "Duplicate cpuset") END_VERSIONS """ } diff --git a/modules/nf-core/bbmap/bbsplit/main.nf b/modules/nf-core/bbmap/bbsplit/main.nf index b55929ce92b..9a0065fcb08 100644 --- a/modules/nf-core/bbmap/bbsplit/main.nf +++ b/modules/nf-core/bbmap/bbsplit/main.nf @@ -51,7 +51,7 @@ process BBMAP_BBSPLIT { cat <<-END_VERSIONS > versions.yml "${task.process}": - bbmap: \$(bbversion.sh 2>&1) + bbmap: \$(bbversion.sh | grep -v "Duplicate cpuset") END_VERSIONS """ } else { @@ -80,7 +80,7 @@ process BBMAP_BBSPLIT { cat <<-END_VERSIONS > versions.yml "${task.process}": - bbmap: \$(bbversion.sh 2>&1) + bbmap: \$(bbversion.sh | grep -v "Duplicate cpuset") END_VERSIONS """ } diff --git a/modules/nf-core/bbmap/clumpify/main.nf b/modules/nf-core/bbmap/clumpify/main.nf index 107e91702ec..102d00c274c 100644 --- a/modules/nf-core/bbmap/clumpify/main.nf +++ b/modules/nf-core/bbmap/clumpify/main.nf @@ -32,7 +32,7 @@ process BBMAP_CLUMPIFY { &> ${prefix}.clumpify.log cat <<-END_VERSIONS > versions.yml "${task.process}": - bbmap: \$(bbversion.sh) + bbmap: \$(bbversion.sh | grep -v "Duplicate cpuset") END_VERSIONS """ } diff --git a/modules/nf-core/bbmap/index/main.nf b/modules/nf-core/bbmap/index/main.nf index fdd6d102008..0ea1add7e4f 100644 --- a/modules/nf-core/bbmap/index/main.nf +++ b/modules/nf-core/bbmap/index/main.nf @@ -28,7 +28,7 @@ process BBMAP_INDEX { cat <<-END_VERSIONS > versions.yml "${task.process}": - bbmap: \$(bbversion.sh) + bbmap: \$(bbversion.sh | grep -v "Duplicate cpuset") END_VERSIONS """ } diff --git a/modules/nf-core/bbmap/pileup/main.nf b/modules/nf-core/bbmap/pileup/main.nf index 1f34efc5640..8463e8beeb8 100644 --- a/modules/nf-core/bbmap/pileup/main.nf +++ b/modules/nf-core/bbmap/pileup/main.nf @@ -31,7 +31,7 @@ process BBMAP_PILEUP { cat <<-END_VERSIONS > versions.yml "${task.process}": - bbmap: \$(bbversion.sh) + bbmap: \$(bbversion.sh | grep -v "Duplicate cpuset") samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) END_VERSIONS From ad13c7a3e125b9e769d94d9558f8526257e4216f Mon Sep 17 00:00:00 2001 From: SPearce Date: Tue, 2 May 2023 14:00:10 +0100 Subject: [PATCH 2/8] Add config for subworkflow --- .../nf-core/bam_ngscheckmate/nextflow.config | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 subworkflows/nf-core/bam_ngscheckmate/nextflow.config diff --git a/subworkflows/nf-core/bam_ngscheckmate/nextflow.config b/subworkflows/nf-core/bam_ngscheckmate/nextflow.config new file mode 100644 index 00000000000..cad9f57cc74 --- /dev/null +++ b/subworkflows/nf-core/bam_ngscheckmate/nextflow.config @@ -0,0 +1,13 @@ +// IMPORTANT: Add this configuration to your modules.config + +process { + withName: ".*BAM_NGSCHECKMATE:BCFTOOLS_MPILEUP" { + ext.args2 = '--no-version --ploidy 1 -c' + ext.args3 = '--no-version' + } + + withName: ".*BAM_NGSCHECKMATE:NGSCHECKMATE_NCM" { + ext.args = '-V' + } + +} From adf50fd44c082dce7f0b0321d31ab6efb49432be Mon Sep 17 00:00:00 2001 From: SPearce Date: Fri, 7 Jul 2023 14:31:58 +0100 Subject: [PATCH 3/8] Update md5sum module to work with multiple files --- modules/nf-core/md5sum/main.nf | 12 +++++++++--- modules/nf-core/md5sum/meta.yml | 8 ++++---- tests/modules/nf-core/md5sum/main.nf | 11 +++++++++++ tests/modules/nf-core/md5sum/test.yml | 13 ++++++++++++- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/modules/nf-core/md5sum/main.nf b/modules/nf-core/md5sum/main.nf index 189f7e9b1e9..e77f69da010 100644 --- a/modules/nf-core/md5sum/main.nf +++ b/modules/nf-core/md5sum/main.nf @@ -8,7 +8,7 @@ process MD5SUM { 'nf-core/ubuntu:20.04' }" input: - tuple val(meta), path(file) + tuple val(meta), file(files) output: tuple val(meta), path("*.md5"), emit: checksum @@ -22,10 +22,16 @@ process MD5SUM { def prefix = task.ext.prefix ?: "${meta.id}" """ + IFS=\$(echo -en "\n\b") + for FILE in $files + do + md5sum \\ $args \\ - ${file} \\ - > ${file}.md5 + \${FILE} \\ + > "\${FILE}.md5" + + done cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/md5sum/meta.yml b/modules/nf-core/md5sum/meta.yml index 0c9d196f9c2..dd6912e3d51 100644 --- a/modules/nf-core/md5sum/meta.yml +++ b/modules/nf-core/md5sum/meta.yml @@ -1,12 +1,12 @@ name: "md5sum" -description: Create an MD5 (128-bit) checksum +description: Create MD5 (128-bit) checksums keywords: - checksum - MD5 - 128 bit tools: - "md5sum": - description: Create an MD5 (128-bit) checksum + description: Create MD5 (128-bit) checksums for each file homepage: "https://www.gnu.org" documentation: "https://man7.org/linux/man-pages/man1/md5sum.1.html" licence: GPLv3+ @@ -17,9 +17,9 @@ input: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - file: + - files: type: file - description: Any file + description: Any number of files. One md5sum file will be generated for each. pattern: "*.*" output: diff --git a/tests/modules/nf-core/md5sum/main.nf b/tests/modules/nf-core/md5sum/main.nf index 56b22acee1e..3b92508177c 100644 --- a/tests/modules/nf-core/md5sum/main.nf +++ b/tests/modules/nf-core/md5sum/main.nf @@ -13,3 +13,14 @@ workflow test_md5sum { MD5SUM ( input ) } + +workflow test_md5sum_pair { + + input = [ + [ id:'test', single_end:false ], // meta map + [ file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true)] + ] + + MD5SUM ( input ) +} diff --git a/tests/modules/nf-core/md5sum/test.yml b/tests/modules/nf-core/md5sum/test.yml index 02b8efe5a28..006ab53a485 100644 --- a/tests/modules/nf-core/md5sum/test.yml +++ b/tests/modules/nf-core/md5sum/test.yml @@ -1,8 +1,19 @@ - name: md5sum test_md5sum - command: nextflow run ./tests/modules/nf-core/md5sum -entry test_md5sum -c ./tests/config/nextflow.config + command: nextflow run ./tests/modules/nf-core/md5sum -entry test_md5sum -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/md5sum/nextflow.config tags: - md5sum files: - path: output/md5sum/test.paired_end.bam.md5 md5sum: 1163095be8fdfb2acb3cc6c027389c4b - path: output/md5sum/versions.yml + +- name: md5sum test_md5sum_pair + command: nextflow run ./tests/modules/nf-core/md5sum -entry test_md5sum_pair -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/md5sum/nextflow.config + tags: + - md5sum + files: + - path: output/md5sum/test.paired_end.bam.md5 + md5sum: 1163095be8fdfb2acb3cc6c027389c4b + - path: output/md5sum/test.single_end.bam.md5 + md5sum: f2d063430925dee72d7d40a1caec0b4e + - path: output/md5sum/versions.yml From 6ff4b9ce19dcb70e81280a99b86eefbd7750407c Mon Sep 17 00:00:00 2001 From: SPearce Date: Fri, 7 Jul 2023 14:36:46 +0100 Subject: [PATCH 4/8] linting --- modules/nf-core/md5sum/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/md5sum/main.nf b/modules/nf-core/md5sum/main.nf index e77f69da010..32e100b589d 100644 --- a/modules/nf-core/md5sum/main.nf +++ b/modules/nf-core/md5sum/main.nf @@ -23,14 +23,14 @@ process MD5SUM { """ IFS=\$(echo -en "\n\b") - for FILE in $files + for FILE in $files do md5sum \\ $args \\ \${FILE} \\ > "\${FILE}.md5" - + done cat <<-END_VERSIONS > versions.yml From 309793ac937ee7c2db5990f92405a1971874d2a9 Mon Sep 17 00:00:00 2001 From: Simon Pearce <24893913+SPPearce@users.noreply.github.com> Date: Sat, 22 Jun 2024 21:46:13 +0000 Subject: [PATCH 5/8] Fix stub --- modules/nf-core/md5sum/main.nf | 8 +- modules/nf-core/md5sum/tests/main.nf.test | 71 +++++++++- .../nf-core/md5sum/tests/main.nf.test.snap | 123 +++++++++++++++++- 3 files changed, 193 insertions(+), 9 deletions(-) diff --git a/modules/nf-core/md5sum/main.nf b/modules/nf-core/md5sum/main.nf index b6a9e448a1d..6916bb627ad 100644 --- a/modules/nf-core/md5sum/main.nf +++ b/modules/nf-core/md5sum/main.nf @@ -8,7 +8,7 @@ process MD5SUM { 'nf-core/ubuntu:20.04' }" input: - tuple val(meta), file(files) + tuple val(meta), path(files) output: tuple val(meta), path("*.md5"), emit: checksum @@ -42,7 +42,11 @@ process MD5SUM { stub: def args = task.ext.args ?: '' """ - touch ${file}.md5 + IFS=\$(echo -en "\n\b") + for FILE in $files + do + touch "\${FILE}.md5" + done cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/md5sum/tests/main.nf.test b/modules/nf-core/md5sum/tests/main.nf.test index 01bda99f599..457e82058d9 100644 --- a/modules/nf-core/md5sum/tests/main.nf.test +++ b/modules/nf-core/md5sum/tests/main.nf.test @@ -51,5 +51,74 @@ nextflow_process { } } } - + + test("md5sum on hello.txt - stub") { + options "-stub" + when { + process { + """ + input[0] = [ + [ id: 'hello' ], + [ file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true) ] + ] + """ + } + } + + then { + assertAll { + { assert process.success } + { assert snapshot(process.out).match() } + } + } + } + + test("md5sum on paired fastq files") { + + when { + process { + """ + input[0] = [ + [ id: 'test' ], + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ] + """ + } + } + + then { + assertAll { + { assert process.success } + { assert snapshot(process.out).match() } + } + } + } + + test("md5sum on paired fastq files - stub") { + options "-stub" + when { + process { + """ + input[0] = [ + [ id: 'test' ], + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ] + """ + } + } + + then { + assertAll { + { assert process.success } + { assert snapshot(process.out).match() } + } + } + } + } diff --git a/modules/nf-core/md5sum/tests/main.nf.test.snap b/modules/nf-core/md5sum/tests/main.nf.test.snap index 66075b44755..555b2f2a12c 100644 --- a/modules/nf-core/md5sum/tests/main.nf.test.snap +++ b/modules/nf-core/md5sum/tests/main.nf.test.snap @@ -1,4 +1,76 @@ { + "md5sum on paired fastq files": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + [ + "test_1.fastq.gz.md5:md5,63a3f187d06ae5cffe77fcd39bbdd63a", + "test_2.fastq.gz.md5:md5,055dc46636836dbd1d9ede2e6ce8cd4e" + ] + ] + ], + "1": [ + "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + ], + "checksum": [ + [ + { + "id": "test" + }, + [ + "test_1.fastq.gz.md5:md5,63a3f187d06ae5cffe77fcd39bbdd63a", + "test_2.fastq.gz.md5:md5,055dc46636836dbd1d9ede2e6ce8cd4e" + ] + ] + ], + "versions": [ + "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-06-22T21:31:08.183442155" + }, + "md5sum on hello.txt - stub": { + "content": [ + { + "0": [ + [ + { + "id": "hello" + }, + "hello.txt.md5:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + ], + "checksum": [ + [ + { + "id": "hello" + }, + "hello.txt.md5:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-06-22T21:36:42.599746839" + }, "md5sum on hello.txt": { "content": [ { @@ -11,7 +83,7 @@ ] ], "1": [ - "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" + "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" ], "checksum": [ [ @@ -22,7 +94,7 @@ ] ], "versions": [ - "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" + "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" ] } ], @@ -30,7 +102,7 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-03-20T17:20:27.185700981" + "timestamp": "2024-06-22T21:27:06.010595726" }, "md5sum on hello.txt (BSD-style)": { "content": [ @@ -44,7 +116,7 @@ ] ], "1": [ - "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" + "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" ], "checksum": [ [ @@ -55,7 +127,46 @@ ] ], "versions": [ - "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" + "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-06-22T21:27:17.173005643" + }, + "md5sum on paired fastq files - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + [ + "test_1.fastq.gz.md5:md5,d41d8cd98f00b204e9800998ecf8427e", + "test_2.fastq.gz.md5:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + ], + "checksum": [ + [ + { + "id": "test" + }, + [ + "test_1.fastq.gz.md5:md5,d41d8cd98f00b204e9800998ecf8427e", + "test_2.fastq.gz.md5:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "versions": [ + "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" ] } ], @@ -63,6 +174,6 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-03-20T17:20:30.00082986" + "timestamp": "2024-06-22T21:37:03.582579201" } } \ No newline at end of file From 6c9733d0219a57e19af89db37cfff3115ebb0205 Mon Sep 17 00:00:00 2001 From: Simon Pearce <24893913+SPPearce@users.noreply.github.com> Date: Mon, 1 Jul 2024 11:10:28 +0000 Subject: [PATCH 6/8] Add option to generate single file --- modules/nf-core/md5sum/main.nf | 75 ++++++---- modules/nf-core/md5sum/tests/main.nf.test | 60 +++++++- .../nf-core/md5sum/tests/main.nf.test.snap | 136 +++++++++++++----- 3 files changed, 205 insertions(+), 66 deletions(-) diff --git a/modules/nf-core/md5sum/main.nf b/modules/nf-core/md5sum/main.nf index 6916bb627ad..012054e4039 100644 --- a/modules/nf-core/md5sum/main.nf +++ b/modules/nf-core/md5sum/main.nf @@ -9,6 +9,7 @@ process MD5SUM { input: tuple val(meta), path(files) + val as_separate_files output: tuple val(meta), path("*.md5"), emit: checksum @@ -19,39 +20,55 @@ process MD5SUM { script: def args = task.ext.args ?: '' - def prefix = task.ext.prefix ?: "${meta.id}" - - """ - IFS=\$(echo -en "\n\b") - for FILE in $files - do - - md5sum \\ - $args \\ - \${FILE} \\ - > "\${FILE}.md5" + def prefix = task.ext.prefix ?: "${meta.id}" // will only use when as_separate_files = false + if ( as_separate_files ) { + """ + find -L * -type f \\ + ! -name '*.md5' \\ + -exec sh -c 'md5sum $args "\$1" > "\$1.md5"' _ "{}" \\; - done + cat <<-END_VERSIONS > versions.yml + "${task.process}": + md5sum: \$(echo \$(md5sum --version 2>&1 | head -n 1| sed 's/^.*) //;' )) + END_VERSIONS + """ + } else { + """ + find -L * -type f \\ + ! -name '*.md5' \\ + -exec md5sum $args "{}" + \\ + > ${prefix}.md5 - cat <<-END_VERSIONS > versions.yml - "${task.process}": - md5sum: \$(echo \$(md5sum --version 2>&1 | head -n 1| sed 's/^.*) //;' )) - END_VERSIONS - """ + cat <<-END_VERSIONS > versions.yml + "${task.process}": + md5sum: \$(echo \$(md5sum --version 2>&1 | head -n 1| sed 's/^.*) //;' )) + END_VERSIONS + """ + } stub: def args = task.ext.args ?: '' - """ - IFS=\$(echo -en "\n\b") - for FILE in $files - do - touch "\${FILE}.md5" - done - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - md5sum: \$(echo \$(md5sum --version 2>&1 | head -n 1| sed 's/^.*) //;' )) - END_VERSIONS - """ + def prefix = task.ext.prefix ?: "${meta.id}" + if ( as_separate_files ) { + """ + find -L * -type f \\ + ! -name '*.md5' \\ + -exec sh -c 'touch "\$1.md5"' _ "{}" \\; + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + md5sum: \$(echo \$(md5sum --version 2>&1 | head -n 1| sed 's/^.*) //;' )) + END_VERSIONS + """ + } else { + """ + touch ${prefix}.md5 + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + md5sum: \$(echo \$(md5sum --version 2>&1 | head -n 1| sed 's/^.*) //;' )) + END_VERSIONS + """ + } } diff --git a/modules/nf-core/md5sum/tests/main.nf.test b/modules/nf-core/md5sum/tests/main.nf.test index 457e82058d9..a6f7cf8614b 100644 --- a/modules/nf-core/md5sum/tests/main.nf.test +++ b/modules/nf-core/md5sum/tests/main.nf.test @@ -17,6 +17,7 @@ nextflow_process { [ id: 'hello' ], [ file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true) ] ] + input[1] = true """ } } @@ -40,6 +41,7 @@ nextflow_process { [ id: 'hello' ], [ file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true) ] ] + input[1] = true """ } } @@ -61,6 +63,7 @@ nextflow_process { [ id: 'hello' ], [ file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true) ] ] + input[1] = true """ } } @@ -73,7 +76,7 @@ nextflow_process { } } - test("md5sum on paired fastq files") { + test("md5sum on paired fastq, combined") { when { process { @@ -85,6 +88,7 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) ] ] + input[1] = false """ } } @@ -97,7 +101,7 @@ nextflow_process { } } - test("md5sum on paired fastq files - stub") { + test("md5sum on paired fastq, combined - stub") { options "-stub" when { process { @@ -109,6 +113,58 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) ] ] + input[1] = false + """ + } + } + + then { + assertAll { + { assert process.success } + { assert snapshot(process.out).match() } + } + } + } + + + test("md5sum on paired fastq, separate") { + + when { + process { + """ + input[0] = [ + [ id: 'test' ], + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ] + input[1] = true + """ + } + } + + then { + assertAll { + { assert process.success } + { assert snapshot(process.out).match() } + } + } + } + + test("md5sum on paired fastq, separate - stub") { + options "-stub" + when { + process { + """ + input[0] = [ + [ id: 'test' ], + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ] + input[1] = true """ } } diff --git a/modules/nf-core/md5sum/tests/main.nf.test.snap b/modules/nf-core/md5sum/tests/main.nf.test.snap index 555b2f2a12c..cd7593b484d 100644 --- a/modules/nf-core/md5sum/tests/main.nf.test.snap +++ b/modules/nf-core/md5sum/tests/main.nf.test.snap @@ -1,5 +1,44 @@ { - "md5sum on paired fastq files": { + "md5sum on paired fastq, separate - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + [ + "test_1.fastq.gz.md5:md5,d41d8cd98f00b204e9800998ecf8427e", + "test_2.fastq.gz.md5:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" + ], + "checksum": [ + [ + { + "id": "test" + }, + [ + "test_1.fastq.gz.md5:md5,d41d8cd98f00b204e9800998ecf8427e", + "test_2.fastq.gz.md5:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "versions": [ + "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.2" + }, + "timestamp": "2024-07-01T11:05:22.111018419" + }, + "md5sum on paired fastq, separate": { "content": [ { "0": [ @@ -14,7 +53,7 @@ ] ], "1": [ - "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" ], "checksum": [ [ @@ -28,15 +67,15 @@ ] ], "versions": [ - "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" ] } ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.2" }, - "timestamp": "2024-06-22T21:31:08.183442155" + "timestamp": "2024-07-01T11:05:12.083864677" }, "md5sum on hello.txt - stub": { "content": [ @@ -50,7 +89,7 @@ ] ], "1": [ - "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" ], "checksum": [ [ @@ -61,15 +100,15 @@ ] ], "versions": [ - "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" ] } ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.2" }, - "timestamp": "2024-06-22T21:36:42.599746839" + "timestamp": "2024-07-01T11:04:42.230073228" }, "md5sum on hello.txt": { "content": [ @@ -83,7 +122,7 @@ ] ], "1": [ - "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" ], "checksum": [ [ @@ -94,50 +133,50 @@ ] ], "versions": [ - "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" ] } ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.2" }, - "timestamp": "2024-06-22T21:27:06.010595726" + "timestamp": "2024-07-01T11:04:22.089948509" }, - "md5sum on hello.txt (BSD-style)": { + "md5sum on paired fastq, combined": { "content": [ { "0": [ [ { - "id": "hello" + "id": "test" }, - "hello.txt.md5:md5,152a03f5dc7aa8db6612f63154ecbca2" + "test.md5:md5,dfb98e45cbb77a9a63ae7029aee38bd1" ] ], "1": [ - "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" ], "checksum": [ [ { - "id": "hello" + "id": "test" }, - "hello.txt.md5:md5,152a03f5dc7aa8db6612f63154ecbca2" + "test.md5:md5,dfb98e45cbb77a9a63ae7029aee38bd1" ] ], "versions": [ - "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" ] } ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.2" }, - "timestamp": "2024-06-22T21:27:17.173005643" + "timestamp": "2024-07-01T11:04:52.085480359" }, - "md5sum on paired fastq files - stub": { + "md5sum on paired fastq, combined - stub": { "content": [ { "0": [ @@ -145,35 +184,62 @@ { "id": "test" }, - [ - "test_1.fastq.gz.md5:md5,d41d8cd98f00b204e9800998ecf8427e", - "test_2.fastq.gz.md5:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + "test.md5:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "1": [ - "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" ], "checksum": [ [ { "id": "test" }, - [ - "test_1.fastq.gz.md5:md5,d41d8cd98f00b204e9800998ecf8427e", - "test_2.fastq.gz.md5:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + "test.md5:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.2" + }, + "timestamp": "2024-07-01T11:05:01.935454952" + }, + "md5sum on hello.txt (BSD-style)": { + "content": [ + { + "0": [ + [ + { + "id": "hello" + }, + "hello.txt.md5:md5,152a03f5dc7aa8db6612f63154ecbca2" + ] + ], + "1": [ + "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" + ], + "checksum": [ + [ + { + "id": "hello" + }, + "hello.txt.md5:md5,152a03f5dc7aa8db6612f63154ecbca2" ] ], "versions": [ - "versions.yml:md5,82690e6c4fd5d61b126c5accbb09d9a4" + "versions.yml:md5,5745fe6b917070b6158ade64fcb4aa91" ] } ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.2" }, - "timestamp": "2024-06-22T21:37:03.582579201" + "timestamp": "2024-07-01T11:04:32.078323542" } } \ No newline at end of file From e3758d765d2bda03c51ce74a45bcb4ed97142455 Mon Sep 17 00:00:00 2001 From: Simon Pearce <24893913+SPPearce@users.noreply.github.com> Date: Mon, 1 Jul 2024 13:26:32 +0100 Subject: [PATCH 7/8] Update modules/nf-core/md5sum/main.nf Co-authored-by: Matthieu Muffato --- modules/nf-core/md5sum/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/md5sum/main.nf b/modules/nf-core/md5sum/main.nf index 012054e4039..4f90ac46967 100644 --- a/modules/nf-core/md5sum/main.nf +++ b/modules/nf-core/md5sum/main.nf @@ -23,7 +23,7 @@ process MD5SUM { def prefix = task.ext.prefix ?: "${meta.id}" // will only use when as_separate_files = false if ( as_separate_files ) { """ - find -L * -type f \\ + find -L * -maxdepth 0 -type f \\ ! -name '*.md5' \\ -exec sh -c 'md5sum $args "\$1" > "\$1.md5"' _ "{}" \\; From 25ef266539e0a4e795d37877c286a4ea147ba371 Mon Sep 17 00:00:00 2001 From: Simon Pearce <24893913+SPPearce@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:27:52 +0000 Subject: [PATCH 8/8] Simplify versions processing --- modules/nf-core/md5sum/main.nf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/md5sum/main.nf b/modules/nf-core/md5sum/main.nf index 4f90ac46967..d77bb8ce84c 100644 --- a/modules/nf-core/md5sum/main.nf +++ b/modules/nf-core/md5sum/main.nf @@ -29,7 +29,7 @@ process MD5SUM { cat <<-END_VERSIONS > versions.yml "${task.process}": - md5sum: \$(echo \$(md5sum --version 2>&1 | head -n 1| sed 's/^.*) //;' )) + md5sum: \$( md5sum --version | sed '1!d; s/.* //' ) END_VERSIONS """ } else { @@ -41,7 +41,7 @@ process MD5SUM { cat <<-END_VERSIONS > versions.yml "${task.process}": - md5sum: \$(echo \$(md5sum --version 2>&1 | head -n 1| sed 's/^.*) //;' )) + md5sum: \$( md5sum --version | sed '1!d; s/.* //' ) END_VERSIONS """ } @@ -57,7 +57,7 @@ process MD5SUM { cat <<-END_VERSIONS > versions.yml "${task.process}": - md5sum: \$(echo \$(md5sum --version 2>&1 | head -n 1| sed 's/^.*) //;' )) + md5sum: \$( md5sum --version | sed '1!d; s/.* //' ) END_VERSIONS """ } else { @@ -66,7 +66,7 @@ process MD5SUM { cat <<-END_VERSIONS > versions.yml "${task.process}": - md5sum: \$(echo \$(md5sum --version 2>&1 | head -n 1| sed 's/^.*) //;' )) + md5sum: \$( md5sum --version | sed '1!d; s/.* //' ) END_VERSIONS """ }