Merge branch 'nf-core-template-merge-3.0.1' into nf-core-template-mer…

…ge-3.0.2
nf-core · Oct 11, 2024 · d458ffe · d458ffe
2 parents c143e9c + f1aa3f9
commit d458ffe
Show file tree

Hide file tree

Showing 345 changed files with 21,373 additions and 658 deletions.
diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml
@@ -32,9 +32,6 @@ jobs:
           test $CURRENT_APPROVALS_COUNT -ge 2 || exit 1 # At least 2 approvals are required
       - name: Launch workflow via Seqera Platform
         uses: seqeralabs/action-tower-launch@v2
-        # TODO nf-core: You can customise AWS full pipeline tests as required
-        # Add full size test data (but still relatively small datasets for few samples)
-        # on the `test_full.config` test runs with only one set of parameters
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,6 +10,10 @@ on:
   workflow_dispatch:
 
 env:
+  NFT_DIFF: "pdiff"
+  NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2"
+  NFT_VER: "0.9.0"
+  NFT_WORKDIR: "~"
   NXF_ANSI_LOG: false
   NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
   NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
@@ -20,24 +24,22 @@ concurrency:
 
 jobs:
   test:
-    name: "Run pipeline with test data (${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }})"
+    name: "Test ${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.NXF_VER }} | ${{ matrix.shard }}/5"
     # Only run on push if this is the nf-core dev branch (merged PRs)
     if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnavar') }}"
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         NXF_VER:
           - "24.04.2"
           - "latest-everything"
-        profile:
-          - "conda"
-          - "docker"
-          - "singularity"
-        test_name:
-          - "test"
+        filter: ["pipeline"]
+        # filter: ["process", "workflow", "function", "pipeline"]
+        profile: ["conda", "docker", "singularity"]
+        shard: [1, 2, 3, 4]
         isMaster:
           - ${{ github.base_ref == 'master' }}
-        # Exclude conda and singularity on dev
         exclude:
           - isMaster: false
             profile: "conda"
@@ -46,12 +48,19 @@ jobs:
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        with:
+          fetch-depth: 0
 
       - name: Set up Nextflow
         uses: nf-core/setup-nextflow@v2
         with:
           version: "${{ matrix.NXF_VER }}"
 
+      - name: Set up nf-test
+        uses: nf-core/setup-nf-test@v1
+        with:
+          version: ${{ env.NFT_VER }}
+
       - name: Set up Apptainer
         if: matrix.profile == 'singularity'
         uses: eWaterCycle/setup-apptainer@main
@@ -62,6 +71,18 @@ jobs:
           mkdir -p $NXF_SINGULARITY_CACHEDIR
           mkdir -p $NXF_SINGULARITY_LIBRARYDIR
 
+      - name: Cache pdiff
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4
+        id: cache-pip-pdiff
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-pdiff
+
+      - name: Set up pdiff to see diff between nf-test snapshots
+        run: |
+          python -m pip install --upgrade pip
+          pip install pdiff cryptography
+
       - name: Set up Miniconda
         if: matrix.profile == 'conda'
         uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
@@ -77,9 +98,29 @@ jobs:
           echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
           echo $(realpath python) >> $GITHUB_PATH
 
-      - name: Clean up Disk space
+      - name: Disk space cleanup
         uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
 
-      - name: "Run pipeline with test data ${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }}"
+      - name: "Run tests | ${{ matrix.filter }}_${{ matrix.profile }} | ${{ matrix.shard }}/5"
+        run: |
+          nf-test test \
+              --ci \
+              --debug \
+              --verbose \
+              --junitxml="TEST-${{ matrix.filter }}_${{ matrix.profile }}_${{ matrix.shard }}.xml" \
+              --shard ${{ matrix.shard }}/5 \
+              --changed-since HEAD^ \
+              --follow-dependencies \
+              --profile "+${{ matrix.profile }}" \
+              --filter ${{ matrix.filter }}
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v4
+        if: success() || failure() # always run even if the previous step fails
+        with:
+          report_paths: "TEST-*.xml"
+
+      - name: Clean up
+        if: always()
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_name }},${{ matrix.profile }} --outdir ./results
+          sudo rm -rf /home/ubuntu/tests/
diff --git a/.github/workflows/template_version_comment.yml b/.github/workflows/template_version_comment.yml
@@ -23,7 +23,6 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install nf-core==${{ steps.read_yml.outputs['nf_core_version'] }}
-
       - name: Check nf-core outdated
         id: nf_core_outdated
         run: echo "OUTPUT=$(pip list --outdated | grep nf-core)" >> ${GITHUB_ENV}

diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,4 @@ testing/
 testing*
 *.pyc
 null/
+.nf-test*
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,14 +3,73 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## v2.0.0dev - [date]
+## dev
 
-Initial release of nf-core/rnavar, created with the [nf-core](https://nf-co.re/) template.
+### Added
 
-### `Added`
+- [#116](https://github.com/nf-core/rnavar/pull/116) - Added `unzip` from nf-core modules for working with unzipped fasta and gtf files
+
+### Changed
+
+- [#78](https://github.com/nf-core/rnavar/pull/78) - Add a gvcf file generated by sample
+- [#78](https://github.com/nf-core/rnavar/pull/78) - Merge on exome.bed
+- [#78](https://github.com/nf-core/rnavar/pull/78) - Add option to choose the type exon/transcript/gene of studied feature in GTF
+- [#95](https://github.com/nf-core/rnavar/pull/95) - Template update for nf-core/tools v2.9
+- [#97](https://github.com/nf-core/rnavar/pull/97) - Template update for nf-core/tools v2.10
+- [#109](https://github.com/nf-core/rnavar/pull/109) - Update all modules
+- [#111](https://github.com/nf-core/rnavar/pull/111) - Template update for nf-core/tools v2.11
+- [#117](https://github.com/nf-core/rnavar/pull/117) - Template update for nf-core/tools v2.11.1
+- [#120](https://github.com/nf-core/rnavar/pull/120) - Template update for nf-core/tools v2.12
+- [#140](https://github.com/nf-core/rnavar/pull/140) - Template update for nf-core/tools v2.14.1
+- [#152](https://github.com/nf-core/rnavar/pull/152) - Template update for nf-core/tools v3.0.1
+
+### Fixed
+
+- [#97](https://github.com/nf-core/rnavar/pull/97) - Update all gatk4 modules to disable JVM hotspot
+- [#98](https://github.com/nf-core/rnavar/pull/98) - Restore annotation
+- [#102](https://github.com/nf-core/rnavar/pull/102) - Fix process name SNPEFF_SNPEFF
+- [#105](https://github.com/nf-core/rnavar/pull/105) - Fixing ch_dict
+- [#106](https://github.com/nf-core/rnavar/pull/106) - Fixing vep_cache
+- [#107](https://github.com/nf-core/rnavar/pull/107) - Fixing star index + single read files
+- [#124](https://github.com/nf-core/rnavar/pull/124) - Fixed s3 bucket path in conditional statement for SnpEff cache
+- [#127](https://github.com/nf-core/rnavar/pull/127) - Fixed s3 bucket path in conditional statement for VEP cache
+- [#130](https://github.com/nf-core/rnavar/pull/130) - Added missing "def" in local variables
+- [#132](https://github.com/nf-core/rnavar/pull/132) - Added missing variantcaller key to meta map, to fix null value in publishDir
+- [#136](https://github.com/nf-core/rnavar/pull/136) - Replaced unzip module with gunzip, removed unzip module
+- [#138](https://github.com/nf-core/rnavar/pull/138) - Proper usage of GVCF
+- [#142](https://github.com/nf-core/rnavar/pull/142) - Fix dbsnp channels
+- [#143](https://github.com/nf-core/rnavar/pull/143) - Use `DROP_MISSING_CONTIGS` by default in `GATK4_BEDTOINTERVALLIST`
+- [#144](https://github.com/nf-core/rnavar/pull/144) - Change gatk_vf params from integer to floats
+- [#149](https://github.com/nf-core/rnavar/pull/149) - Updated ch_gtf and ch_fasta_fai channels emitted by main.nf
+
+### Dependencies
 
-### `Fixed`
+| Dependency | Old version | New version |
+| ---------- | ----------- | ----------- |
+| bcftools   | 1.17        | 1.18        |
+| bedtools   | 2.31.0      | 2.31.1      |
+| fastqc     | 0.11.9      | 0.12.1      |
+| mosdepth   | 0.3.3       | 0.3.6       |
+| multiqc    | 1.15        | 1.18        |
+| samtools   | 1.17        | 1.18        |
+
+## [1.0.0] nfcore/rnavar - 2022/06/20
+
+First production release of the pipeline with latest software versions.
+
+This version is based on GATK4 best-practices for RNAseq [Ref](https://github.com/gatk-workflows/gatk4-rnaseq-germline-snps-indels) and it includes:
+
+### `Added`
 
-### `Dependencies`
+- Added `FastQC v0.11.9` from nf-core modules for read-level QC and summary.
+- Added `STAR v2.7.9a` from nf-core modules for read alignment to reference genome.
+- Added `Samtools v1.15.1` from nf-core modules for alignment statistics and QC.
+- Added `GATK v4.2.6.1` from nf-core modules for alignment post-processing, variant calling and filtration.
+- Added `Tabix v1.11` from nf-core modules for indexing BAM ann VCF files.
+- Added `SnpEff v5.0` from nf-core modules for variant annotation.
+- Added `Ensembl VEP v104.3` from nf-core modules for variant annotation.
+- Added `MultiQC v1.12` from nf-core modules for QC summary report.
+- Added Scatter i.e., one interval-list into many interval-files to run multiple processes in parallel.
 
-### `Deprecated`
+Thanks to everyone that contributed to this release.
+Special thanks to @maxulysse and @FriederikeHanssen for your review and valuable suggestions.
diff --git a/CITATIONS.md b/CITATIONS.md
@@ -14,10 +14,38 @@
 
 > Andrews, S. (2010). FastQC: A Quality Control Tool for High Throughput Sequence Data [Online].
 
+- [STAR](https://pubmed.ncbi.nlm.nih.gov/23104886/)
+
+  > Dobin A, Davis CA, Schlesinger F, Drenkow J, Zaleski C, Jha S, Batut P, Chaisson M, Gingeras TR. STAR: ultrafast universal RNA-seq aligner Bioinformatics. 2013 Jan 1;29(1):15-21. doi: 10.1093/bioinformatics/bts635. Epub 2012 Oct 25. PubMed PMID: 23104886; PubMed Central PMCID: PMC3530905.
+
+- [SAMtools](https://pubmed.ncbi.nlm.nih.gov/19505943/)
+
+  > Li H, Handsaker B, Wysoker A, Fennell T, Ruan J, Homer N, Marth G, Abecasis G, Durbin R; 1000 Genome Project Data Processing Subgroup. The Sequence Alignment/Map format and SAMtools. Bioinformatics. 2009 Aug 15;25(16):2078-9. doi: 10.1093/bioinformatics/btp352. Epub 2009 Jun 8. PubMed PMID: 19505943; PubMed Central PMCID: PMC2723002.
+
+- [GATK](https://pubmed.ncbi.nlm.nih.gov/20644199/)
+
+  > McKenna A, Hanna M, Banks E, et al.: The Genome Analysis Toolkit: a MapReduce framework for analyzing next-generation DNA sequencing data. Genome Res. 2010 Sep;20(9):1297-303. doi: 10.1101/gr.107524.110. Epub 2010 Jul 19. PubMed PMID: 20644199; PubMed Central PMCID: PMC2928508.
+
+- [snpEff](https://pubmed.ncbi.nlm.nih.gov/22728672/)
+
+  > Cingolani P, Platts A, Wang le L, et al.: A program for annotating and predicting the effects of single nucleotide polymorphisms, SnpEff: SNPs in the genome of Drosophila melanogaster strain w1118; iso-2; iso-3. Fly (Austin). Apr-Jun 2012;6(2):80-92. doi: 10.4161/fly.19695. PubMed PMID: 22728672; PubMed Central PMCID: PMC3679285.
+
+- [VEP](https://pubmed.ncbi.nlm.nih.gov/27268795/)
+
+  > McLaren W, Gil L, Hunt SE, et al.: The Ensembl Variant Effect Predictor. Genome Biol. 2016 Jun 6;17(1):122. doi: 10.1186/s13059-016-0974-4. PubMed PMID: 27268795; PubMed Central PMCID: PMC4893825.
+
 - [MultiQC](https://pubmed.ncbi.nlm.nih.gov/27312411/)
 
 > Ewels P, Magnusson M, Lundin S, Käller M. MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics. 2016 Oct 1;32(19):3047-8. doi: 10.1093/bioinformatics/btw354. Epub 2016 Jun 16. PubMed PMID: 27312411; PubMed Central PMCID: PMC5039924.
 
+- [Tabix](https://pubmed.ncbi.nlm.nih.gov/21208982/)
+
+  > Heng Li, Tabix: fast retrieval of sequence features from generic TAB-delimited files, Bioinformatics, Volume 27, Issue 5, 1 March 2011, Pages 718–719. doi: 10.1093/bioinformatics/btq671. PubMed PMID: 21208982; PubMed Central PMCID: PMC3042176.
+
+- [R](https://www.R-project.org/)
+
+  > R Core Team (2017). R: A language and environment for statistical computing. R Foundation for Statistical Computing, Vienna, Austria.
+
 ## Software packaging/containerisation tools
 
 - [Anaconda](https://anaconda.com)
-Original file line number
+Diff line change
@@ Expand Up / @@ -7,3 +7,4 @@ testing/ @@
     testing*
     *.pyc
     null/
+    .nf-test*