DSL2+

.github/workflows/ci.yml

@@ -59,7 +59,7 @@ jobs:
       fail-fast: false
       matrix:
         NXF_VER:
-          - "24.04.2"
+          - "24.10.2"
           - "latest-everything"
         profile:
           - "conda"

README.md

@@ -8,7 +8,7 @@
 [![GitHub Actions CI Status](https://github.com/nf-core/fetchngs/actions/workflows/ci.yml/badge.svg)](https://github.com/nf-core/fetchngs/actions/workflows/ci.yml)
 [![GitHub Actions Linting Status](https://github.com/nf-core/fetchngs/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/fetchngs/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/fetchngs/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.5070524-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.5070524)[![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com)
 
-[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.2-23aa62.svg)](https://www.nextflow.io/)
+[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.10.2-23aa62.svg)](https://www.nextflow.io/)
 [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)
 [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)
 [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)

conf/base.config

@@ -15,12 +15,6 @@ process {
     memory = { 6.GB   * task.attempt }
     time   = { 4.h    * task.attempt }
 
-    publishDir = [
-        path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" },
-        mode: params.publish_dir_mode,
-        saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-    ]
-
     errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' }
     maxRetries    = 1
     maxErrors     = '-1'

main.nf

@@ -9,6 +9,9 @@
 ----------------------------------------------------------------------------------------
 */
 
+nextflow.preview.output = true
+nextflow.preview.topic = true
+
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS / WORKFLOWS
@@ -18,6 +21,7 @@
 include { SRA                     } from './workflows/sra'
 include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_fetchngs_pipeline'
 include { PIPELINE_COMPLETION     } from './subworkflows/local/utils_nfcore_fetchngs_pipeline'
+include { softwareVersionsToYAML  } from './subworkflows/nf-core/utils_nfcore_pipeline'
 
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -38,8 +42,16 @@ workflow NFCORE_FETCHNGS {
     //
     // WORKFLOW: Download FastQ files for SRA / ENA / GEO / DDBJ ids
     //
-    SRA ( ids )
+    SRA (
+        ids,
+        params.ena_metadata_fields ?: '',
+        params.download_method,
+        params.skip_fastq_download,
+        params.dbgap_key
+    )
 
+    emit:
+    samples = SRA.out.samples
 }
 
 /*
@@ -81,6 +93,36 @@ workflow {
         params.monochrome_logs,
         params.hook_url
     )
+
+    publish:
+    NFCORE_FETCHNGS.out.samples >> 'samples'
+    softwareVersionsToYAML() >> 'versions'
+}
+
+
+output {
+    samples {
+        path { _sample ->
+            def dirs = [
+                'fastq': 'fastq',
+                'md5': 'fastq/md5'
+            ]
+            return { filename ->
+                def ext = filename.tokenize('.').last()
+                "${dirs[ext]}/${filename}"
+            }
+        }
+        index {
+            path 'samplesheet/samplesheet.json'
+        }
+    }
+
+    versions {
+        path '.'
+        index {
+            path 'nf_core_fetchngs_software_mqc_versions.yml'
+        }
+    }
 }
 
 /*

modules/local/aspera_cli/main.nf

@@ -14,7 +14,7 @@ process ASPERA_CLI {
     output:
     tuple val(meta), path("*fastq.gz"), emit: fastq
     tuple val(meta), path("*md5")     , emit: md5
-    path "versions.yml"               , emit: versions
+    tuple val("${task.process}"), val('aspera_cli'), eval('ascli --version'), topic: versions
 
     script:
     def args = task.ext.args ?: ''
@@ -31,11 +31,6 @@ process ASPERA_CLI {
 
         echo "${meta.md5_1}  ${meta.id}.fastq.gz" > ${meta.id}.fastq.gz.md5
         md5sum -c ${meta.id}.fastq.gz.md5
-
-        cat <<-END_VERSIONS > versions.yml
-        "${task.process}":
-            aspera_cli: \$(ascli --version)
-        END_VERSIONS
         """
     } else {
         """
@@ -58,11 +53,22 @@ process ASPERA_CLI {
 
         echo "${meta.md5_2}  ${meta.id}_2.fastq.gz" > ${meta.id}_2.fastq.gz.md5
         md5sum -c ${meta.id}_2.fastq.gz.md5
-
-        cat <<-END_VERSIONS > versions.yml
-        "${task.process}":
-            aspera_cli: \$(ascli --version)
-        END_VERSIONS
         """
     }
 }
+
+workflow {
+    def input = [
+        [ id:'SRX9626017_SRR13191702', single_end:false, md5_1: '89c5be920021a035084d8aeb74f32df7', md5_2: '56271be38a80db78ef3bdfc5d9909b98' ],
+        [
+            'fasp.sra.ebi.ac.uk:/vol1/fastq/SRR131/002/SRR13191702/SRR13191702_1.fastq.gz',
+            'fasp.sra.ebi.ac.uk:/vol1/fastq/SRR131/002/SRR13191702/SRR13191702_2.fastq.gz'
+        ]
+    ]
+    def user = 'era-fasp'
+
+    ASPERA_CLI (
+        input,
+        user
+    )
+}

modules/local/aspera_cli/nextflow.config

@@ -1,17 +1,5 @@
 process {
     withName: 'ASPERA_CLI' {
         ext.args = '-QT -l 300m -P33001'
-        publishDir = [
-            [
-                path: { "${params.outdir}/fastq" },
-                mode: params.publish_dir_mode,
-                pattern: "*.fastq.gz"
-            ],
-            [
-                path: { "${params.outdir}/fastq/md5" },
-                mode: params.publish_dir_mode,
-                pattern: "*.md5"
-            ]
-        ]
     }
 }

modules/local/multiqc_mappings_config/main.nf

@@ -11,17 +11,12 @@ process MULTIQC_MAPPINGS_CONFIG {
 
     output:
     path "*config.yml" , emit: yml
-    path "versions.yml", emit: versions
+    tuple val("${task.process}"), val('python'), eval("python --version | sed 's/Python //g'"), topic: versions
 
     script:
     """
     multiqc_mappings_config.py \\
         $csv \\
         multiqc_config.yml
-
-    cat <<-END_VERSIONS > versions.yml
-    "${task.process}":
-        python: \$(python --version | sed 's/Python //g')
-    END_VERSIONS
     """
 }

modules/local/sra_fastq_ftp/main.nf

@@ -15,7 +15,7 @@ process SRA_FASTQ_FTP {
     output:
     tuple val(meta), path("*fastq.gz"), emit: fastq
     tuple val(meta), path("*md5")     , emit: md5
-    path "versions.yml"               , emit: versions
+    tuple val("${task.process}"), val('wget'), eval("echo \$(wget --version | head -n 1 | sed 's/^GNU Wget //; s/ .*\$//')"), topic: versions
 
     script:
     def args = task.ext.args ?: ''
@@ -28,11 +28,6 @@ process SRA_FASTQ_FTP {
 
         echo "${meta.md5_1}  ${meta.id}.fastq.gz" > ${meta.id}.fastq.gz.md5
         md5sum -c ${meta.id}.fastq.gz.md5
-
-        cat <<-END_VERSIONS > versions.yml
-        "${task.process}":
-            wget: \$(echo \$(wget --version | head -n 1 | sed 's/^GNU Wget //; s/ .*\$//'))
-        END_VERSIONS
         """
     } else {
         """
@@ -51,11 +46,6 @@ process SRA_FASTQ_FTP {
 
         echo "${meta.md5_2}  ${meta.id}_2.fastq.gz" > ${meta.id}_2.fastq.gz.md5
         md5sum -c ${meta.id}_2.fastq.gz.md5
-
-        cat <<-END_VERSIONS > versions.yml
-        "${task.process}":
-            wget: \$(echo \$(wget --version | head -n 1 | sed 's/^GNU Wget //; s/ .*\$//'))
-        END_VERSIONS
         """
     }
 }

modules/local/sra_fastq_ftp/nextflow.config

@@ -1,17 +1,5 @@
 process {
     withName: 'SRA_FASTQ_FTP' {
         ext.args = '-t 5 -nv -c -T 60'
-        publishDir = [
-            [
-                path: { "${params.outdir}/fastq" },
-                mode: params.publish_dir_mode,
-                pattern: "*.fastq.gz"
-            ],
-            [
-                path: { "${params.outdir}/fastq/md5" },
-                mode: params.publish_dir_mode,
-                pattern: "*.md5"
-            ]
-        ]
     }
 }

modules/local/sra_ids_to_runinfo/main.nf

@@ -14,7 +14,7 @@ process SRA_IDS_TO_RUNINFO {
 
     output:
     path "*.tsv"       , emit: tsv
-    path "versions.yml", emit: versions
+    tuple val("${task.process}"), val('python'), eval("python --version | sed 's/Python //g'"), topic: versions
 
     script:
     def metadata_fields = fields ? "--ena_metadata_fields ${fields}" : ''
@@ -24,10 +24,5 @@ process SRA_IDS_TO_RUNINFO {
         id.txt \\
         ${id}.runinfo.tsv \\
         $metadata_fields
-
-    cat <<-END_VERSIONS > versions.yml
-    "${task.process}":
-        python: \$(python --version | sed 's/Python //g')
-    END_VERSIONS
     """
 }

modules/local/sra_runinfo_to_ftp/main.nf

@@ -11,17 +11,12 @@ process SRA_RUNINFO_TO_FTP {
 
     output:
     path "*.tsv"       , emit: tsv
-    path "versions.yml", emit: versions
+    tuple val("${task.process}"), val('python'), eval("python --version | sed 's/Python //g'"), topic: versions
 
     script:
     """
     sra_runinfo_to_ftp.py \\
         ${runinfo.join(',')} \\
         ${runinfo.toString().tokenize(".")[0]}.runinfo_ftp.tsv
-
-    cat <<-END_VERSIONS > versions.yml
-    "${task.process}":
-        python: \$(python --version | sed 's/Python //g')
-    END_VERSIONS
     """
 }

nextflow.config

@@ -33,6 +33,7 @@ params {
     version                      = false
     modules_testdata_base_path   = 's3://ngi-igenomes/testdata/nf-core/modules/'
     pipelines_testdata_base_path = 's3://ngi-igenomes/testdata/nf-core/pipelines/fetchngs/1.15.0/'
+    trace_report_suffix          = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')
 
     // Config options
     config_profile_name          = null
@@ -197,22 +198,25 @@ set -C # No clobber - prevent output redirection from overwriting files.
 // Disable process selector warnings by default. Use debug profile to enable warnings.
 nextflow.enable.configProcessNamesValidation = false
 
-def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')
+// workflow outputs
+outputDir = params.outdir
+workflow.output.mode = params.publish_dir_mode
+
 timeline {
     enabled = true
-    file    = "${params.outdir}/pipeline_info/execution_timeline_${trace_timestamp}.html"
+    file    = "${params.outdir}/pipeline_info/execution_timeline_${params.trace_report_suffix}.html"
 }
 report {
     enabled = true
-    file    = "${params.outdir}/pipeline_info/execution_report_${trace_timestamp}.html"
+    file    = "${params.outdir}/pipeline_info/execution_report_${params.trace_report_suffix}.html"
 }
 trace {
     enabled = true
-    file    = "${params.outdir}/pipeline_info/execution_trace_${trace_timestamp}.txt"
+    file    = "${params.outdir}/pipeline_info/execution_trace_${params.trace_report_suffix}.txt"
 }
 dag {
     enabled = true
-    file    = "${params.outdir}/pipeline_info/pipeline_dag_${trace_timestamp}.html"
+    file    = "${params.outdir}/pipeline_info/pipeline_dag_${params.trace_report_suffix}.html"
 }
 
 manifest {
@@ -221,7 +225,7 @@ manifest {
     homePage        = 'https://github.com/nf-core/fetchngs'
     description     = """Pipeline to fetch metadata and raw FastQ files from public databases"""
     mainScript      = 'main.nf'
-    nextflowVersion = '!>=24.04.2'
+    nextflowVersion = '!>=24.10.2'
     version         = '1.13.0dev'
     doi             = '10.5281/zenodo.5070524'
 }

nextflow_schema.json

@@ -199,6 +199,12 @@
                     "description": "Base URL or local path to location of pipeline test dataset files",
                     "default": "s3://ngi-igenomes/testdata/nf-core/pipelines/fetchngs/1.15.0/",
                     "hidden": true
+                },
+                "trace_report_suffix": {
+                    "type": "string",
+                    "fa_icon": "far calendar",
+                    "description": "Suffix to add to the trace report filename. Default is the date and time in the format yyyy-MM-dd_HH-mm-ss.",
+                    "hidden": true
                 }
             }
         },

workflows/sra/main.nf

@@ -9,8 +9,6 @@ include { SRA_FASTQ_FTP           } from '../../modules/local/sra_fastq_ftp'
 include { SRA_IDS_TO_RUNINFO      } from '../../modules/local/sra_ids_to_runinfo'
 include { SRA_RUNINFO_TO_FTP      } from '../../modules/local/sra_runinfo_to_ftp'
 include { ASPERA_CLI              } from '../../modules/local/aspera_cli'
-include { SRA_TO_SAMPLESHEET      } from '../../modules/local/sra_to_samplesheet'
-include { softwareVersionsToYAML  } from '../../subworkflows/nf-core/utils_nfcore_pipeline'
 
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -29,27 +27,27 @@ include { FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS } from '../../subworkflow
 workflow SRA {
 
     take:
-    ids // channel: [ ids ]
+    ids                 // channel: [ ids ]
+    ena_metadata_fields // string
+    download_method     // enum: 'aspera' | 'ftp' | 'sratools'
+    skip_fastq_download // boolean
+    dbgap_key           // string
 
     main:
-    ch_versions = Channel.empty()
-
     //
     // MODULE: Get SRA run information for public database ids
     //
     SRA_IDS_TO_RUNINFO (
         ids,
-        params.ena_metadata_fields ?: ''
+        ena_metadata_fields
     )
-    ch_versions = ch_versions.mix(SRA_IDS_TO_RUNINFO.out.versions.first())
 
     //
     // MODULE: Parse SRA run information, create file containing FTP links and read into workflow as [ meta, [reads] ]
     //
     SRA_RUNINFO_TO_FTP (
         SRA_IDS_TO_RUNINFO.out.tsv
     )
-    ch_versions = ch_versions.mix(SRA_RUNINFO_TO_FTP.out.versions.first())
 
     SRA_RUNINFO_TO_FTP
         .out
@@ -64,27 +62,27 @@ workflow SRA {
         .unique()
         .set { ch_sra_metadata }
 
-    if (!params.skip_fastq_download) {
+    if (!skip_fastq_download) {
 
         ch_sra_metadata
             .branch {
                 meta ->
-                    def download_method = 'ftp'
+                    def method = 'ftp'
                     // meta.fastq_aspera is a metadata string with ENA fasp links supported by Aspera
                         // For single-end: 'fasp.sra.ebi.ac.uk:/vol1/fastq/ERR116/006/ERR1160846/ERR1160846.fastq.gz'
                         // For paired-end: 'fasp.sra.ebi.ac.uk:/vol1/fastq/SRR130/020/SRR13055520/SRR13055520_1.fastq.gz;fasp.sra.ebi.ac.uk:/vol1/fastq/SRR130/020/SRR13055520/SRR13055520_2.fastq.gz'
-                    if (meta.fastq_aspera && params.download_method == 'aspera') {
-                        download_method = 'aspera'
+                    if (meta.fastq_aspera && download_method == 'aspera') {
+                        method = 'aspera'
                     }
-                    if ((!meta.fastq_aspera && !meta.fastq_1) || params.download_method == 'sratools') {
-                        download_method = 'sratools'
+                    if ((!meta.fastq_aspera && !meta.fastq_1) || download_method == 'sratools') {
+                        method = 'sratools'
                     }
 
-                    aspera: download_method == 'aspera'
+                    aspera: method == 'aspera'
                         return [ meta, meta.fastq_aspera.tokenize(';').take(2) ]
-                    ftp: download_method == 'ftp'
+                    ftp: method == 'ftp'
                         return [ meta, [ meta.fastq_1, meta.fastq_2 ] ]
-                    sratools: download_method == 'sratools'
+                    sratools: method == 'sratools'
                         return [ meta, meta.run_accession ]
             }
             .set { ch_sra_reads }
@@ -95,16 +93,14 @@ workflow SRA {
         SRA_FASTQ_FTP (
             ch_sra_reads.ftp
         )
-        ch_versions = ch_versions.mix(SRA_FASTQ_FTP.out.versions.first())
 
         //
         // SUBWORKFLOW: Download sequencing reads without FTP links using sra-tools.
         //
         FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS (
             ch_sra_reads.sratools,
-            params.dbgap_key ? file(params.dbgap_key, checkIfExists: true) : []
+            dbgap_key ? file(dbgap_key, checkIfExists: true) : []
         )
-        ch_versions = ch_versions.mix(FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.versions.first())
 
         //
         // MODULE: If Aspera link is provided in run information then download FastQ directly via Aspera CLI and validate with md5sums
@@ -113,80 +109,37 @@ workflow SRA {
             ch_sra_reads.aspera,
             'era-fasp'
         )
-        ch_versions = ch_versions.mix(ASPERA_CLI.out.versions.first())
 
         // Isolate FASTQ channel which will be added to emit block
-        SRA_FASTQ_FTP
-            .out
-            .fastq
+        ch_fastq = SRA_FASTQ_FTP.out.fastq
             .mix(FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.reads)
             .mix(ASPERA_CLI.out.fastq)
-            .map {
-                meta, fastq ->
-                    def reads = fastq instanceof List ? fastq.flatten() : [ fastq ]
-                    def meta_clone = meta.clone()
 
-                    meta_clone.fastq_1 = reads[0] ? "${params.outdir}/fastq/${reads[0].getName()}" : ''
-                    meta_clone.fastq_2 = reads[1] && !meta.single_end ? "${params.outdir}/fastq/${reads[1].getName()}" : ''
+        ch_md5 = SRA_FASTQ_FTP.out.md5
+            .mix(ASPERA_CLI.out.md5)
 
-                    return meta_clone
+        ch_samples = ch_fastq
+            .join(ch_md5, remainder: true)
+            .map {
+                meta, fastq, md5 ->
+                    def reads = fastq instanceof List ? fastq.flatten() : [ fastq ]
+                    meta + [
+                        fastq_1: reads[0],
+                        fastq_2: reads[1] && !meta.single_end ? reads[1] : null,
+                        md5_1: md5[0],
+                        md5_2: md5[1] && !meta.single_end ? md5[1] : null,
+                    ]
             }
-            .set { ch_sra_metadata }
     }
-
-    //
-    // MODULE: Stage FastQ files downloaded by SRA together and auto-create a samplesheet
-    //
-    SRA_TO_SAMPLESHEET (
-        ch_sra_metadata,
-        params.nf_core_pipeline ?: '',
-        params.nf_core_rnaseq_strandedness ?: 'auto',
-        params.sample_mapping_fields
-    )
-
-    // Merge samplesheets and mapping files across all samples
-    SRA_TO_SAMPLESHEET
-        .out
-        .samplesheet
-        .map { it[1] }
-        .collectFile(name:'tmp_samplesheet.csv', newLine: true, keepHeader: true, sort: { it.baseName })
-        .map { it.text.tokenize('\n').join('\n') }
-        .collectFile(name:'samplesheet.csv', storeDir: "${params.outdir}/samplesheet")
-        .set { ch_samplesheet }
-
-    SRA_TO_SAMPLESHEET
-        .out
-        .mappings
-        .map { it[1] }
-        .collectFile(name:'tmp_id_mappings.csv', newLine: true, keepHeader: true, sort: { it.baseName })
-        .map { it.text.tokenize('\n').join('\n') }
-        .collectFile(name:'id_mappings.csv', storeDir: "${params.outdir}/samplesheet")
-        .set { ch_mappings }
-
-    //
-    // MODULE: Create a MutiQC config file with sample name mappings
-    //
-    ch_sample_mappings_yml = Channel.empty()
-    if (params.sample_mapping_fields) {
-        MULTIQC_MAPPINGS_CONFIG (
-            ch_mappings
-        )
-        ch_versions = ch_versions.mix(MULTIQC_MAPPINGS_CONFIG.out.versions)
-        ch_sample_mappings_yml = MULTIQC_MAPPINGS_CONFIG.out.yml
+    else {
+        ch_samples = Channel.empty()
     }
 
-    //
-    // Collate and save software versions
-    //
-    softwareVersionsToYAML(ch_versions)
-        .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_fetchngs_software_mqc_versions.yml', sort: true, newLine: true)
-
     emit:
-    samplesheet     = ch_samplesheet
-    mappings        = ch_mappings
-    sample_mappings = ch_sample_mappings_yml
-    sra_metadata    = ch_sra_metadata
-    versions        = ch_versions.unique()
+    samples = ch_samples
+
+    publish:
+    SRA_RUNINFO_TO_FTP.out.tsv  >> 'metadata'
 }
 
 /*

workflows/sra/nextflow.config

@@ -1,8 +1,3 @@
-includeConfig "../../modules/local/multiqc_mappings_config/nextflow.config"
 includeConfig "../../modules/local/aspera_cli/nextflow.config"
 includeConfig "../../modules/local/sra_fastq_ftp/nextflow.config"
-includeConfig "../../modules/local/sra_ids_to_runinfo/nextflow.config"
-includeConfig "../../modules/local/sra_runinfo_to_ftp/nextflow.config"
-includeConfig "../../modules/local/sra_to_samplesheet/nextflow.config"
-includeConfig "../../modules/nf-core/sratools/prefetch/nextflow.config"
 includeConfig "../../subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config"