Merge pull request #182 from nf-core/fix/restructured-nf-tests

Fix/restructured nf tests
nf-core · Aug 16, 2023 · 23d2f21 · 23d2f21
2 parents 6cd8d3f + 2011565
commit 23d2f21
Show file tree

Hide file tree

Showing 53 changed files with 1,210 additions and 474 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -40,8 +40,19 @@ jobs:
     needs: define_nxf_versions
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         NXF_VER: ${{ fromJson(needs.define_nxf_versions.outputs.matrix) }}
+        test_tags:
+          - "sra"
+          - "sra_force_sratools_download"
+          - "sra_skip_fastq"
+          - "sra_with_atacseq"
+          - "sra_with_rnaseq"
+          - "sra_with_rnaseq_specific_ena"
+          - "sra_with_taxprofiler"
+          - "sra_with_viralrecon"
+          - "modules_local"
         profile:
           - "docker"
 
@@ -61,7 +72,7 @@ jobs:
 
       - name: Run nf-test
         run: |
-          nf-test test tests/pipeline/ --profile "test,${{ matrix.profile }}" --junitxml=test.xml
+          nf-test test --tag ${{ matrix.test_tags }} --profile "test,${{ matrix.profile }}" --junitxml=test.xml
 
       - name: Output log on failure
         if: failure()

diff --git a/.nf-core.yml b/.nf-core.yml
@@ -1,7 +1,15 @@
 repository_type: pipeline
 lint:
   files_unchanged:
+    - .github/CONTRIBUTING.md
+    - .github/ISSUE_TEMPLATE/bug_report.yml
+    - assets/nf-core-fetchngs_logo_light.png
     - assets/sendmail_template.txt
     - lib/NfcoreTemplate.groovy
     - .gitattributes
-  actions_ci: False
+  actions_ci: false
+  multiqc_config: false
+  nextflow_config:
+    - "params.validationShowHiddenParams"
+    - "params.validationSchemaIgnoreParams"
+  schema_params: false
diff --git a/conf/test.config b/conf/test.config
@@ -22,3 +22,6 @@ params {
     // Input data
     input  = 'https://raw.githubusercontent.com/nf-core/test-datasets/fetchngs/sra_ids_test.csv'
 }
+
+// Load test_data.config for nf-tests
+includeConfig 'test_data.config'
diff --git a/conf/test_data.config b/conf/test_data.config
@@ -0,0 +1,36 @@
+params {
+    // Base directory for test data
+    test_data_base = "https://raw.githubusercontent.com/nf-core/test-datasets/fetchngs"
+
+    merge_samplesheet_ids = [ "DRX024467_DRR026872", "SRX11047067_SRR14709033", "SRX9504942_SRR13055517", "DRX026011_DRR028935", "SRX17709227_SRR21711856", "SRX9504942_SRR13055518", "ERX1188904_ERR1109373", "SRX17709228_SRR21711855", "SRX9504942_SRR13055519", "ERX1234253_ERR1160846", "SRX6725035_SRR9984183", "SRX9504942_SRR13055520", "SRX10940790_SRR14593545", "SRX9315476_SRR12848126", "SRX9626017_SRR13191702" ]
+
+    def merge_samplesheet_url  = "${params.test_data_base}/modules/local/sra_merge_samplesheet/samplesheets/"
+    def merge_mappings_url     = "${params.test_data_base}/modules/local/sra_merge_samplesheet/mappings/"
+    def merge_samplesheet_urls = []
+    def merge_mappings_urls    = []
+
+    merge_samplesheet_ids.each { id ->
+        merge_samplesheet_urls += "${merge_samplesheet_url}${id}.samplesheet.csv"
+        merge_mappings_urls    += "${merge_mappings_url}/${id}.mappings.csv"
+    }
+
+    fastq_ftp_ids      = ["SRR13191702"]
+    def fastq_ftp_url  = "ftp.sra.ebi.ac.uk/vol1/fastq/SRR131/002/SRR13191702/"
+    def fastq_ftp_urls = []
+
+    fastq_ftp_ids.each { id ->
+        fastq_ftp_urls += "${fastq_ftp_url}${id}_1.fastq.gz"
+        fastq_ftp_urls += "${fastq_ftp_url}${id}_2.fastq.gz"
+    }
+
+    test_data {
+        'modules_local' {
+                multiqc_mappings_config             = "${params.test_data_base}/modules/local/multiqc_mappings_config/SRX9626017_SRR13191702.mappings.csv"
+                sra_merge_samplesheet_samplesheets  = merge_samplesheet_urls
+                sra_merge_samplesheet_mappings      = merge_mappings_urls
+                sra_to_samplesheet                  = "${params.test_data_base}/modules/local/sra_to_samplesheet/SRX9626017_SRR13191702.mappings.csv"
+                sra_fastq_ftp                       = fastq_ftp_urls
+                sra_runinfo_to_ftp                  = "${params.test_data_base}/modules/local/sra_runinfo_to_ftp/SRR13191702.runinfo.tsv"
+        }
+    }
+}
diff --git a/lib/NftestUtils.groovy b/lib/NftestUtils.groovy
@@ -0,0 +1,38 @@
+// Helper functions for pipeline tests
+
+class NftestUtils {
+
+    // Function to remove Nextflow version from software_versions.yml
+    public static String removeNextflowVersion(outputDir) {
+        def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml
+        if (softwareVersions.containsKey("Workflow")) {
+            softwareVersions.Workflow.remove("Nextflow")
+        }
+        return softwareVersions
+    }
+
+    // Function to filter lines from a file and return a new file
+    public static File filterLines(String inFilePath, int linesToSkip) {
+        if (linesToSkip >= 0) {
+            File inputFile = new File(inFilePath)
+            File outputFile = new File(inFilePath + ".filtered")
+            def lineCount = 0
+            inputFile.eachLine { line ->
+                lineCount++
+                if (lineCount > linesToSkip) {
+                    outputFile.append(line + '\n')
+                }
+            }
+            return outputFile
+        } else {
+            File inputFile = new File(inFilePath)
+            File outputFile = new File(inFilePath + ".filtered")
+            def lines = inputFile.readLines()
+            def totalLines = lines.size()
+            lines.take(totalLines + linesToSkip).each { line ->
+                outputFile.append(line + '\n')
+            }
+            return outputFile
+        }
+    }
+}
diff --git a/main.nf b/main.nf
@@ -49,9 +49,9 @@ if (WorkflowMain.isSraId(ch_input)) {
 
 if (params.input_type == input_type) {
     if (params.input_type == 'sra') {
-        include { SRA } from './workflows/sra'
+        include { SRA } from './workflows/sra/main'
     } else if (params.input_type == 'synapse') {
-        include { SYNAPSE } from './workflows/synapse'
+        include { SYNAPSE } from './workflows/synapse/main'
     }
 } else {
     exit 1, "Ids auto-detected as ${input_type}. Please provide '--input_type ${input_type}' as a parameter to the pipeline!"

diff --git a/modules/local/multiqc_mappings_config.nf → ...appings_config/multiqc_mappings_config.nf b/modules/local/multiqc_mappings_config.nf → ...appings_config/multiqc_mappings_config.nf
diff --git a/modules/local/multiqc_mappings_config/multiqc_mappings_config.nf.test b/modules/local/multiqc_mappings_config/multiqc_mappings_config.nf.test
@@ -0,0 +1,31 @@
+nextflow_process {
+
+    name "Test Process MULTIQC_MAPPINGS_CONFIG"
+    script "modules/local/multiqc_mappings_config/multiqc_mappings_config.nf"
+    process "MULTIQC_MAPPINGS_CONFIG"
+    tag "multiqc_mappings_config"
+    tag "modules_local"
+
+    test("SRX9626017_SRR13191702") {
+
+        when {
+            params {
+                outdir   = "$outputDir"
+            }
+            process {
+                """
+                input[0] = file(params.test_data['modules_local']['multiqc_mappings_config'], checkIfExists: true)
+                """
+            }
+        }
+
+        then {
+            assertAll(
+                { assert process.success },
+                { assert snapshot(process.out).match() },
+            )
+        }
+
+    }
+
+}
diff --git a/modules/local/multiqc_mappings_config/multiqc_mappings_config.nf.test.snap b/modules/local/multiqc_mappings_config/multiqc_mappings_config.nf.test.snap
@@ -0,0 +1,27 @@
+{
+    "SRX9626017_SRR13191702": {
+        "content": [
+            {
+                "0": [
+                    [
+                        "multiqc_config.yml:md5,7f3cb10fff83ba9eb3e8fa6862d1290a",
+                        "versions.yml:md5,dd4c66f0551d15510b36bb2e2b2fdd73"
+                    ]
+                ],
+                "1": [
+                    "versions.yml:md5,dd4c66f0551d15510b36bb2e2b2fdd73"
+                ],
+                "versions": [
+                    "versions.yml:md5,dd4c66f0551d15510b36bb2e2b2fdd73"
+                ],
+                "yml": [
+                    [
+                        "multiqc_config.yml:md5,7f3cb10fff83ba9eb3e8fa6862d1290a",
+                        "versions.yml:md5,dd4c66f0551d15510b36bb2e2b2fdd73"
+                    ]
+                ]
+            }
+        ],
+        "timestamp": "2023-07-05T15:01:18+0000"
+    }
+}
diff --git a/modules/local/sra_fastq_ftp.nf → modules/local/sra_fastq_ftp/sra_fastq_ftp.nf b/modules/local/sra_fastq_ftp.nf → modules/local/sra_fastq_ftp/sra_fastq_ftp.nf
diff --git a/modules/local/sra_fastq_ftp/sra_fastq_ftp.nf.test b/modules/local/sra_fastq_ftp/sra_fastq_ftp.nf.test
@@ -0,0 +1,34 @@
+nextflow_process {
+
+    name "Test Process SRA_FASTQ_FTP"
+    script "modules/local/sra_fastq_ftp/sra_fastq_ftp.nf"
+    process "SRA_FASTQ_FTP"
+    tag "sra_fastq_ftp"
+    tag "modules_local"
+
+    test("SRX9626017_SRR13191702") {
+
+        when {
+            params {
+                outdir   = "$outputDir"
+            }
+            process {
+                """
+                input[0] = [
+                            [ id:'SRX9626017_SRR13191702', single_end:false, md5_1: '89c5be920021a035084d8aeb74f32df7', md5_2: '56271be38a80db78ef3bdfc5d9909b98' ], // meta map
+                            params.test_data['modules_local']['sra_fastq_ftp']
+                        ]
+                """
+            }
+        }
+
+        then {
+            assertAll(
+                { assert process.success },
+                { assert snapshot(process.out).match() },
+            )
+        }
+
+    }
+
+}
diff --git a/modules/local/sra_fastq_ftp/sra_fastq_ftp.nf.test.snap b/modules/local/sra_fastq_ftp/sra_fastq_ftp.nf.test.snap
@@ -0,0 +1,140 @@
+{
+    "SRX9626017_SRR13191702": {
+        "content": [
+            {
+                "0": [
+                    [
+                        {
+                            "id": "SRX9626017_SRR13191702",
+                            "single_end": false,
+                            "md5_1": "89c5be920021a035084d8aeb74f32df7",
+                            "md5_2": "56271be38a80db78ef3bdfc5d9909b98"
+                        },
+                        [
+                            "SRX9626017_SRR13191702_1.fastq.gz:md5,89c5be920021a035084d8aeb74f32df7",
+                            "SRX9626017_SRR13191702_2.fastq.gz:md5,56271be38a80db78ef3bdfc5d9909b98"
+                        ]
+                    ]
+                ],
+                "1": [
+                    [
+                        {
+                            "id": "SRX9626017_SRR13191702",
+                            "single_end": false,
+                            "md5_1": "89c5be920021a035084d8aeb74f32df7",
+                            "md5_2": "56271be38a80db78ef3bdfc5d9909b98"
+                        },
+                        [
+                            "SRX9626017_SRR13191702_1.fastq.gz.md5:md5,055a6916ec9ee478e453d50651f87997",
+                            "SRX9626017_SRR13191702_2.fastq.gz.md5:md5,c30ac785f8d80ec563fabf604d8bf945"
+                        ]
+                    ]
+                ],
+                "2": [
+                    "versions.yml:md5,6b9d69dea1c1305f74a65197ee871f1b"
+                ],
+                "fastq": [
+                    [
+                        {
+                            "id": "SRX9626017_SRR13191702",
+                            "single_end": false,
+                            "md5_1": "89c5be920021a035084d8aeb74f32df7",
+                            "md5_2": "56271be38a80db78ef3bdfc5d9909b98"
+                        },
+                        [
+                            "SRX9626017_SRR13191702_1.fastq.gz:md5,89c5be920021a035084d8aeb74f32df7",
+                            "SRX9626017_SRR13191702_2.fastq.gz:md5,56271be38a80db78ef3bdfc5d9909b98"
+                        ]
+                    ]
+                ],
+                "md5": [
+                    [
+                        {
+                            "id": "SRX9626017_SRR13191702",
+                            "single_end": false,
+                            "md5_1": "89c5be920021a035084d8aeb74f32df7",
+                            "md5_2": "56271be38a80db78ef3bdfc5d9909b98"
+                        },
+                        [
+                            "SRX9626017_SRR13191702_1.fastq.gz.md5:md5,055a6916ec9ee478e453d50651f87997",
+                            "SRX9626017_SRR13191702_2.fastq.gz.md5:md5,c30ac785f8d80ec563fabf604d8bf945"
+                        ]
+                    ]
+                ],
+                "versions": [
+                    "versions.yml:md5,6b9d69dea1c1305f74a65197ee871f1b"
+                ]
+            }
+        ],
+        "timestamp": "2023-08-06T23:31:57+0000"
+    },
+    "Should run without failures": {
+        "content": [
+            {
+                "0": [
+                    [
+                        {
+                            "id": "SRX9626017_SRR13191702",
+                            "single_end": false,
+                            "md5_1": "89c5be920021a035084d8aeb74f32df7",
+                            "md5_2": "56271be38a80db78ef3bdfc5d9909b98"
+                        },
+                        [
+                            "SRX9626017_SRR13191702_1.fastq.gz:md5,89c5be920021a035084d8aeb74f32df7",
+                            "SRX9626017_SRR13191702_2.fastq.gz:md5,56271be38a80db78ef3bdfc5d9909b98"
+                        ]
+                    ]
+                ],
+                "1": [
+                    [
+                        {
+                            "id": "SRX9626017_SRR13191702",
+                            "single_end": false,
+                            "md5_1": "89c5be920021a035084d8aeb74f32df7",
+                            "md5_2": "56271be38a80db78ef3bdfc5d9909b98"
+                        },
+                        [
+                            "SRX9626017_SRR13191702_1.fastq.gz.md5:md5,055a6916ec9ee478e453d50651f87997",
+                            "SRX9626017_SRR13191702_2.fastq.gz.md5:md5,c30ac785f8d80ec563fabf604d8bf945"
+                        ]
+                    ]
+                ],
+                "2": [
+                    "versions.yml:md5,6b9d69dea1c1305f74a65197ee871f1b"
+                ],
+                "fastq": [
+                    [
+                        {
+                            "id": "SRX9626017_SRR13191702",
+                            "single_end": false,
+                            "md5_1": "89c5be920021a035084d8aeb74f32df7",
+                            "md5_2": "56271be38a80db78ef3bdfc5d9909b98"
+                        },
+                        [
+                            "SRX9626017_SRR13191702_1.fastq.gz:md5,89c5be920021a035084d8aeb74f32df7",
+                            "SRX9626017_SRR13191702_2.fastq.gz:md5,56271be38a80db78ef3bdfc5d9909b98"
+                        ]
+                    ]
+                ],
+                "md5": [
+                    [
+                        {
+                            "id": "SRX9626017_SRR13191702",
+                            "single_end": false,
+                            "md5_1": "89c5be920021a035084d8aeb74f32df7",
+                            "md5_2": "56271be38a80db78ef3bdfc5d9909b98"
+                        },
+                        [
+                            "SRX9626017_SRR13191702_1.fastq.gz.md5:md5,055a6916ec9ee478e453d50651f87997",
+                            "SRX9626017_SRR13191702_2.fastq.gz.md5:md5,c30ac785f8d80ec563fabf604d8bf945"
+                        ]
+                    ]
+                ],
+                "versions": [
+                    "versions.yml:md5,6b9d69dea1c1305f74a65197ee871f1b"
+                ]
+            }
+        ],
+        "timestamp": "2023-08-06T23:31:57+0000"
+    }
+}
diff --git a/modules/local/sra_ids_to_runinfo.nf → .../sra_ids_to_runinfo/sra_ids_to_runinfo.nf b/modules/local/sra_ids_to_runinfo.nf → .../sra_ids_to_runinfo/sra_ids_to_runinfo.nf