Nf composition (#14)

* initial commit

* Prokka and RATT running

* generating plots and KEGG pathway views

* Full workflow, with explicit outputs and TODOs resolved.

* Containerized

* comments

* Nf phage finder (#9)

Merging two workflows with related functionality.

* proof of concept: linked runAnnotation and phageFinder, with phageFinder able to be turned on or off

* restructured early pipeline. added nf-test infrastructure

* testing added for sra2fastq

* Testing for the first few modules

* integrated hostRemoval

* runAssembly in pipeline

* host removal testing + cleanup

* tests for runAssembly

* reads to contig restructured as subworkflow. adding nf-test CI

* fixing JDK version for CI

* testing Apptainer for CI

* cleanup

* updated snapshots for JDK 17

* adding debugging output

* more testing output for GH actions

* added Git LFS to testing yml

* adding sharding to tests - checking to see if this resolves space issues with the runner

* removing files from LFS

* basic testing for runReadsToContig

* adding nf-test file

* attempting optimized testing requirements

* reverting testing strategy

* host removal testing accounts for inconsistent file naming

Author: aw-watson

sra2fastq/__init__.py .gitattributes

@@ -0,0 +1,39 @@
+name: CI Tests
+
+on: [push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        shard: [1, 2, 3, 4]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v2
+        with:
+          java-version: '17'
+          distribution: 'adopt'
+
+      - name: Set up Apptainer 1.3.0
+        uses: eWaterCycle/setup-apptainer@v2
+        with:
+          apptainer-version: 1.3.0
+
+
+      - name: Setup Nextflow 24.10.1
+        uses: nf-core/setup-nextflow@v1
+        with:
+          version: "24.10.1"
+
+      - name: Install nf-test
+        run: |
+          wget -qO- https://get.nf-test.com | bash
+          sudo mv nf-test /usr/local/bin/
+
+      - name: Run Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }})
+        run: nf-test test --ci --shard ${{ matrix.shard }}/${{ strategy.job-total }}

.github/workflows/ci-tests.yml

@@ -1,5 +1,6 @@
 **/__pycache__/*
 .nextflow*
-*/work/*
+**/work/**
 */logs/*
-**/ec_info/*
+**/ec_info/*
+.nf-test*

.gitignore

@@ -0,0 +1,53 @@
+#!/usr/bin/env nextflow
+
+include {SRA2FASTQ} from './modules/sra2fastq/sra2fastq.nf'
+include {COUNTFASTQ} from './modules/countFastq/countFastq.nf'
+include {FAQCS} from './modules/runFaQCs/runFaQCs.nf'
+include {HOSTREMOVAL} from './modules/hostRemoval/hostRemoval.nf'
+include {ASSEMBLY} from './modules/runAssembly/runAssembly.nf'
+include {READSTOCONTIGS} from './modules/runReadsToContig/runReadsToContig.nf'
+
+workflow {
+
+    //input specification
+
+    pairedFiles = channel.fromPath(params.pairedFiles, checkIfExists:true)
+    unpairedFiles = channel.fromPath(params.unpairedFiles, checkIfExists:true)
+    contigs = channel.empty()
+    if(params.r2c.useAssembledContigs) {
+        contigs = channel.fromPath(params.inputContigs, checkIfExists:true)
+    }
+
+    if(params.modules.sra2fastq) {
+        SRA2FASTQ(params.sra2fastq.plus(params.shared))
+        pairedFiles = pairedFiles.concat(SRA2FASTQ.out.paired).flatten()
+        unpairedFiles = unpairedFiles.concat(SRA2FASTQ.out.unpaired).flatten()
+    }
+    
+    COUNTFASTQ(pairedFiles.collect(), unpairedFiles.collect())
+
+    avgLen = COUNTFASTQ.out.avgReadLen
+    paired = COUNTFASTQ.out.paired.ifEmpty(params.pairedFiles)
+    unpaired = COUNTFASTQ.out.unpaired.ifEmpty(params.unpairedFiles)
+
+
+    if(params.modules.faqcs) {
+        FAQCS(params.faqcs.plus(params.shared),paired,unpaired,avgLen)
+        paired = FAQCS.out.paired.ifEmpty(params.pairedFiles)
+        unpaired = FAQCS.out.unpaired.ifEmpty(params.unpairedFiles)
+    }
+
+    if(params.modules.hostRemoval) {
+        HOSTREMOVAL(params.hostRemoval.plus(params.shared),paired,unpaired)
+        paired = HOSTREMOVAL.out.paired.ifEmpty(params.pairedFiles)
+        unpaired = HOSTREMOVAL.out.unpaired.ifEmpty(params.unpairedFiles)
+    }
+
+    if(params.modules.runAssembly && !params.r2c.useAssembledContigs) {
+        ASSEMBLY(params.assembly.plus(params.shared), paired, unpaired, avgLen)
+        contigs = ASSEMBLY.out.outContigs
+        READSTOCONTIGS(params.r2c.plus(params.shared), paired, unpaired, contigs)
+    }
+
+
+}

hostRemoval/hostRemoval.nf

@@ -0,0 +1,37 @@
+# syntax=docker/dockerfile:1
+FROM continuumio/miniconda3:24.5.0-0 AS build
+
+ENV container=docker
+
+# add conda channels
+RUN conda config --add channels conda-forge \
+    && conda config --add channels bioconda
+
+RUN conda init bash \
+    && . ~/.bashrc \
+    && conda create --name countFastq \
+    && conda activate countFastq 
+
+RUN conda install -n countFastq -c conda-forge perl
+
+RUN conda install -c conda-forge conda-pack
+
+#add scripts from this project to bin
+ADD bin/* /opt/conda/envs/countFastq/bin
+
+#pack environment for runtime image
+RUN conda-pack -n countFastq -o /tmp/env.tar && \
+    mkdir /venv && cd /venv && tar xf /tmp/env.tar && \
+    rm /tmp/env.tar
+
+RUN /venv/bin/conda-unpack
+
+FROM debian:latest AS runtime
+
+COPY --from=build /venv /venv
+
+ENV PATH=/venv/bin:$PATH
+RUN apt-get update && apt-get install procps -y && apt-get clean
+
+SHELL ["/bin/bash", "-c"]
+CMD /bin/bash