From ff84e6570dc4cc90eb897e63b6867284468bf8be Mon Sep 17 00:00:00 2001 From: whalleyt Date: Wed, 6 Mar 2024 13:02:07 +0000 Subject: [PATCH] update tm01 --- README.md | 19 ++- run_test.sh | 10 +- .../docker/Dockerfile.clockwork-0.9.8 | 120 ++++++++++++++++ .../docker/Dockerfile.preprocessing-0.9.8 | 131 ++++++++++++++++++ .../docker/Dockerfile.tbprofiler-0.9.8 | 54 ++++++++ .../docker/Dockerfile.vcfpredict-0.9.8 | 27 ++++ .../docker/docker/Dockerfile.clockwork-0.9.8 | 120 ++++++++++++++++ .../docker/Dockerfile.preprocessing-0.9.8 | 131 ++++++++++++++++++ .../docker/docker/Dockerfile.tbprofiler-0.9.8 | 54 ++++++++ .../docker/docker/Dockerfile.vcfpredict-0.9.8 | 27 ++++ test_scripts/mainscripts/TM01_main.nf | 51 +------ .../config/config/containers.config | 48 +++++++ .../mainscripts/config/containers.config | 48 +++++++ .../docker/Dockerfile.clockwork-0.9.8 | 120 ++++++++++++++++ .../docker/Dockerfile.preprocessing-0.9.8 | 131 ++++++++++++++++++ .../docker/Dockerfile.tbprofiler-0.9.8 | 54 ++++++++ .../docker/Dockerfile.vcfpredict-0.9.8 | 27 ++++ test_scripts/mainscripts/nextflow.config | 124 +++++++++++++++++ test_scripts/workflows/TM01.nf | 34 ++--- 19 files changed, 1255 insertions(+), 75 deletions(-) create mode 100644 test_scripts/docker/Dockerfile.clockwork-0.9.8 create mode 100644 test_scripts/docker/Dockerfile.preprocessing-0.9.8 create mode 100644 test_scripts/docker/Dockerfile.tbprofiler-0.9.8 create mode 100644 test_scripts/docker/Dockerfile.vcfpredict-0.9.8 create mode 100644 test_scripts/docker/docker/Dockerfile.clockwork-0.9.8 create mode 100644 test_scripts/docker/docker/Dockerfile.preprocessing-0.9.8 create mode 100644 test_scripts/docker/docker/Dockerfile.tbprofiler-0.9.8 create mode 100644 test_scripts/docker/docker/Dockerfile.vcfpredict-0.9.8 create mode 100644 test_scripts/mainscripts/config/config/containers.config create mode 100644 test_scripts/mainscripts/config/containers.config create mode 100644 test_scripts/mainscripts/docker/Dockerfile.clockwork-0.9.8 create mode 100644 test_scripts/mainscripts/docker/Dockerfile.preprocessing-0.9.8 create mode 100644 test_scripts/mainscripts/docker/Dockerfile.tbprofiler-0.9.8 create mode 100644 test_scripts/mainscripts/docker/Dockerfile.vcfpredict-0.9.8 create mode 100644 test_scripts/mainscripts/nextflow.config diff --git a/README.md b/README.md index aebede0..361abc8 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Lodestone Testing ## Aims -This repositry is designed to perform automated testing of [Lodestone](https://github.com/Pathogen-Genomics-Cymru/tb-pipeline) pipeline. It takes test datasets help in an [S3 bucket](https://s3.console.aws.amazon.com/s3/buckets/sp3testdata), downloads them along with the requisite Kraken2 and Bowtie databases and runs the nextflow pipeline against them based on changes to the module module files. +This repositry is designed to perform automated testing of [Lodestone](https://github.com/Pathogen-Genomics-Cymru/lodestone) pipeline. It takes test datasets help in an [S3 bucket](https://microbial-bioin-sp3.s3.climb.ac.uk/Lodestone_Testing_1.0/), downloads them along with the requisite Kraken2 and Bowtie databases and runs the nextflow pipeline against them based on changes to the module module files. ## Usage This repo is primarily intended to be automated. However, the test scripts can still be ran as follows: @@ -11,11 +11,18 @@ This repo is primarily intended to be automated. However, the test scripts can s Where ```_test_ids``` refers to the test ID, ranging from TM01 to TM10. Multiple arguments can be supplied. Additional arguments can be supplied to point to the correct Kraken2 and Bowtie databases, as well to run the script with or without singularity. See ```bash run_test.sh -h``` for more information: ``` -bash run_test.h -t -k -b -i - -t : string of of test IDs. These can be TM[01-10]. - -k : Kraken2 database directory. Default = pluspf_16gb - -b : Bowtie database directory. Default = hg19_1kgmaj - -i : Bowtie index prefix. Default = hg19_1kgmaj +bash run_test.h -t -k -b -i -a -r + -t : string of of test IDs. These can be TM[01-10]. + Multiple arguments can be supplied but they must each have the -t flag before + them e.g. -t TM01 -t TM02 + -d : Directory containing test data. Default = s3://microbial-bioin-sp3/Lodestone_Testing_1.0/ + -k : Kraken2 database directory. Default = + -b : Bowtie database directory. Default = + -i : Bowtie index prefix. Default = + -a : Afanc database directory. Default = + -r : Resource directory. Default = + -p : Nextflow profile. Default = climb + ``` ## Output diff --git a/run_test.sh b/run_test.sh index 0b973fd..5b23806 100644 --- a/run_test.sh +++ b/run_test.sh @@ -44,6 +44,10 @@ done shift $((OPTIND -1)) #recursively copy and update any scripts into new location so we don't have to override basedir in NF +cp -R -u -p lodestone/nextflow.config test_scripts/mainscripts +cp -R -u -p lodestone/config test_scripts/mainscripts +cp -R -u -p lodestone/singularity test_scripts/mainscripts +cp -R -u -p lodestone/docker test_scripts/mainscripts cp -R -u -p lodestone/bin test_scripts/mainscripts ###### @@ -76,9 +80,9 @@ fi if [[ $profile == "" ]]; then #go to our default - profile="--profile climb" + profile="-profile climb" else - profile="--profile $profile" + profile="-profile $profile" fi if [[ $data == "" ]]; then @@ -97,7 +101,7 @@ $bowtie_db $bowtie_index $kraken_db $profile --pattern '*_{1,2}.fq.gz' -with-rep for id in "${test_args[@]}"; do #set input and output and find test script script="test_scripts/mainscripts/${id}_main.nf" - input_dir="$data/$id" + input_dir="${data}${id}" output_dir="${id}_out" #run it diff --git a/test_scripts/docker/Dockerfile.clockwork-0.9.8 b/test_scripts/docker/Dockerfile.clockwork-0.9.8 new file mode 100644 index 0000000..d45b6c4 --- /dev/null +++ b/test_scripts/docker/Dockerfile.clockwork-0.9.8 @@ -0,0 +1,120 @@ +FROM debian:buster + +LABEL maintainer="pricea35@cardiff.ac.uk" \ +about.summary="container for the clockwork workflow" + +ENV samtools_version=1.12 \ +htslib_version=1.12 \ +bcftools_version=1.12 \ +minimap2_version=2.17 \ +picard_version=2.18.16 \ +gramtools_version=8af53f6c8c0d72ef95223e89ab82119b717044f2 \ +vt_version=2187ff6347086e38f71bd9f8ca622cd7dcfbb40c \ +minos_version=0.11.0 \ +cortex_version=3a235272e4e0121be64527f01e73f9e066d378d3 \ +vcftools_version=0.1.15 \ +mccortex_version=97aba198d632ee98ac1aa496db33d1a7a8cb7e51 \ +stampy_version=1.0.32r3761 \ +python_version=3.6.5 \ +clockwork_version=2364dec4cbf25c844575e19e8fe0a319d10721b5 + +ENV PACKAGES="procps curl git build-essential wget zlib1g-dev pkg-config jq r-base-core rsync autoconf libncurses-dev libbz2-dev liblzma-dev libcurl4-openssl-dev cmake tabix libvcflib-tools libssl-dev software-properties-common perl locales locales-all" \ +PYTHON="python2.7 python-dev" + +COPY bin/ /opt/bin/ +ENV PATH=/opt/bin:$PATH + + +RUN apt-get update \ +&& apt-get install -y $PACKAGES $PYTHON \ +&& curl -fsSL https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tgz | tar -xz \ +&& cd Python-${python_version} \ +&& ./configure --enable-optimizations \ +&& make altinstall \ +&& cd .. \ +&& ln -s /usr/local/bin/python3.6 /usr/local/bin/python3 \ +&& ln -s /usr/local/bin/pip3.6 /usr/local/bin/pip3 \ +&& pip3 install --upgrade pip \ +&& pip3 install 'cluster_vcf_records==0.13.1' pysam setuptools awscli \ +&& wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - \ +&& add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ \ +&& apt-get update && apt-get install -y adoptopenjdk-8-hotspot + + +RUN curl -fsSL https://github.com/samtools/samtools/archive/${samtools_version}.tar.gz | tar -xz \ +&& curl -fsSL https://github.com/samtools/htslib/releases/download/${htslib_version}/htslib-${htslib_version}.tar.bz2 | tar -xj \ +&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} \ +&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ +&& rm -r samtools-${samtools_version} \ +&& curl -fsSL https://github.com/samtools/bcftools/archive/refs/tags/${bcftools_version}.tar.gz | tar -xz \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ +&& rm -r bcftools-${bcftools_version} + + +RUN curl -fsSL minimap2-${minimap2_version}.tar.gz https://github.com/lh3/minimap2/archive/v${minimap2_version}.tar.gz | tar -xz \ +&& cd minimap2-${minimap2_version} \ +&& make \ +&& chmod +x minimap2 \ +&& mv minimap2 /usr/local/bin \ +&& cd .. \ +&& rm -r minimap2-${minimap2_version} \ +&& wget https://github.com/broadinstitute/picard/releases/download/${picard_version}/picard.jar -O /usr/local/bin/picard.jar + + +RUN git clone https://github.com/atks/vt.git vt-git \ +&& cd vt-git \ +&& git checkout ${vt_version} \ +&& make \ +&& cd .. \ +&& mv vt-git/vt /usr/local/bin \ +&& pip3 install tox "six>=1.14.0" \ +&& git clone https://github.com/iqbal-lab-org/gramtools \ +&& cd gramtools \ +&& git checkout ${gramtools_version} \ +&& pip3 install . \ +&& cd .. \ +&& pip3 install cython \ +&& pip3 install git+https://github.com/iqbal-lab-org/minos@v${minos_version} + + +RUN git clone --recursive https://github.com/iqbal-lab/cortex.git \ +&& cd cortex \ +&& git checkout ${cortex_version} \ +&& bash install.sh \ +&& make NUM_COLS=1 cortex_var \ +&& make NUM_COLS=2 cortex_var \ +&& cd .. \ +&& mkdir bioinf-tools \ +&& cd bioinf-tools \ +&& curl -fsSL http://www.well.ox.ac.uk/~gerton/software/Stampy/stampy-${stampy_version}.tgz | tar -xz \ +&& make -C stampy-* \ +&& cp -s stampy-*/stampy.py . \ +&& curl -fsSL https://github.com/vcftools/vcftools/releases/download/v${vcftools_version}/vcftools-${vcftools_version}.tar.gz | tar -xz \ +&& cd vcftools-${vcftools_version} \ +&& ./configure --prefix $PWD/install \ +&& make && make install \ +&& ln -s src/perl/ . \ +&& cd .. \ +&& git clone --recursive https://github.com/mcveanlab/mccortex \ +&& cd mccortex \ +&& git checkout ${mccortex_version} \ +&& make all \ +&& cd .. \ +&& cp -s mccortex/bin/mccortex31 . \ +&& cd .. \ +&& git clone https://github.com/iqbal-lab-org/clockwork \ +&& cd clockwork \ +&& git checkout ${clockwork_version} \ +&& cd python \ +&& pip3 install . \ +&& chmod +x scripts/clockwork + +ENV CLOCKWORK_CORTEX_DIR=/cortex \ +PATH=${PATH}:/clockwork/python/scripts \ +PICARD_JAR=/usr/local/bin/picard.jar + +ENV LC_ALL en_US.UTF-8 \ +LANG en_US.UTF-8 \ +LANGUAGE en_US.UTF-8 + diff --git a/test_scripts/docker/Dockerfile.preprocessing-0.9.8 b/test_scripts/docker/Dockerfile.preprocessing-0.9.8 new file mode 100644 index 0000000..1905969 --- /dev/null +++ b/test_scripts/docker/Dockerfile.preprocessing-0.9.8 @@ -0,0 +1,131 @@ +FROM ubuntu:focal + +LABEL maintainer="pricea35@cardiff.ac.uk" \ +about.summary="container for the preprocessing workflow" + +ENV samtools_version=1.12 \ +bcftools_version=1.12 \ +htslib_version=1.12 \ +bedtools_version=2.29.2 \ +bowtie2_version=2.4.2 \ +fastp_version=0.20.1 \ +fastqc_version=0.11.9 \ +fqtools_version=2.3 \ +kraken2_version=2.1.1 \ +afanc_version=0.10.2 \ +mykrobe_version=0.12.1 \ +bwa_version=0.7.17 \ +mash_version=2.3 \ +fastani_version=1.33 + +ENV PACKAGES="procps curl git wget build-essential zlib1g-dev libncurses-dev libz-dev libbz2-dev liblzma-dev libcurl4-openssl-dev libgsl-dev rsync unzip ncbi-blast+ pigz jq libtbb-dev openjdk-11-jre-headless autoconf r-base-core locales locales-all" \ +PYTHON="python3 python3-pip python3-dev" \ +PYTHON_PACKAGES="biopython awscli boto3" + +ENV PATH=${PATH}:/usr/local/bin/mccortex/bin:/usr/local/bin/bwa-${bwa_version}:/opt/edirect \ +LD_LIBRARY_PATH=/usr/local/lib + +RUN export DEBIAN_FRONTEND="noninteractive" + +COPY bin/ /opt/bin/ +ENV PATH=/opt/bin:$PATH + +RUN apt-get update \ +&& DEBIAN_FRONTEND="noninteractive" apt-get install -y $PACKAGES $PYTHON \ +&& pip3 install --upgrade pip \ +&& pip3 install $PYTHON_PACKAGES \ +&& ln -s /usr/bin/python3 /usr/bin/python + +RUN curl -fsSL https://github.com/samtools/samtools/archive/${samtools_version}.tar.gz | tar -xz \ +&& curl -fsSL https://github.com/samtools/htslib/releases/download/${htslib_version}/htslib-${htslib_version}.tar.bz2 | tar -xj \ +&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} \ +&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ +&& rm -r samtools-${samtools_version} \ +&& curl -fsSL https://github.com/samtools/bcftools/archive/refs/tags/${bcftools_version}.tar.gz | tar -xz \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ +&& rm -r bcftools-${bcftools_version} + +RUN curl -fsSL https://github.com/alastair-droop/fqtools/archive/v${fqtools_version}.tar.gz | tar -xz \ +&& mv htslib-${htslib_version} fqtools-${fqtools_version} \ +&& cd fqtools-${fqtools_version} \ +&& mv htslib-${htslib_version} htslib \ +&& cd htslib \ +&& autoreconf -i \ +&& ./configure \ +&& make \ +&& make install \ +&& cd .. \ +&& make \ +&& mv bin/* /usr/local/bin \ +&& chmod +x /usr/local/bin/fqtools \ +&& cd .. \ +&& rm -r fqtools-${fqtools_version} + +RUN curl -fsSL https://github.com/arq5x/bedtools2/releases/download/v${bedtools_version}/bedtools-${bedtools_version}.tar.gz | tar -xz \ +&& make -C bedtools2 \ +&& mv bedtools2/bin/* /usr/local/bin \ +&& rm -r bedtools2 + +RUN curl -fsSL https://sourceforge.net/projects/bowtie-bio/files/bowtie2/${bowtie2_version}/bowtie2-${bowtie2_version}-source.zip -o bowtie2-${bowtie2_version}-source.zip \ +&& unzip bowtie2-${bowtie2_version}-source.zip \ +&& make -C bowtie2-${bowtie2_version} prefix=/usr/local install \ +&& rm -r bowtie2-${bowtie2_version} \ +&& rm bowtie2-${bowtie2_version}-source.zip + +RUN curl -fsSL https://github.com/OpenGene/fastp/archive/v${fastp_version}.tar.gz | tar -xz \ +&& cd fastp-${fastp_version} \ +&& make \ +&& make install \ +&& cd .. \ +&& rm -r fastp-${fastp_version} + +RUN wget https://www.bioinformatics.babraham.ac.uk/projects/fastqc/fastqc_v${fastqc_version}.zip \ +&& unzip fastqc_v${fastqc_version}.zip \ +&& chmod +x FastQC/fastqc \ +&& mv FastQC/* /usr/local/bin \ +&& rm fastqc_v${fastqc_version}.zip \ +&& rm -r FastQC + +RUN curl -fsSL https://github.com/DerrickWood/kraken2/archive/v${kraken2_version}.tar.gz | tar -xz \ +&& cd kraken2-${kraken2_version} \ +&& ./install_kraken2.sh /usr/local/bin \ +&& cd .. + +RUN curl -fsSL https://github.com/ArthurVM/Afanc/archive/refs/tags/v${afanc_version}-alpha.tar.gz | tar -xz \ +&& cd Afanc-${afanc_version}-alpha \ +&& pip3 install ./ \ +&& cd .. \ +&& curl -fsSL "https://github.com/marbl/Mash/releases/download/v${mash_version}/mash-Linux64-v${mash_version}.tar" | tar -x \ +&& mv mash-Linux64-v${mash_version}/mash /usr/local/bin \ +&& rm -r mash-Linux* \ +&& wget https://github.com/ParBLiSS/FastANI/releases/download/v${fastani_version}/fastANI-Linux64-v${fastani_version}.zip \ +&& unzip fastANI-Linux64-v${fastani_version}.zip \ +&& mv fastANI /usr/local/bin + +RUN sh -c "$(curl -fsSL ftp://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/install-edirect.sh)" \ +&& mkdir -p /opt/edirect \ +&& mv /root/edirect/* /opt/edirect + +RUN git clone --recursive -b geno_kmer_count https://github.com/phelimb/mccortex \ +&& make -C mccortex \ +&& mv mccortex /usr/local/bin \ +&& curl -fsSL mykrobe-${mykrobe_version}.tar.gz https://github.com/Mykrobe-tools/mykrobe/archive/v${mykrobe_version}.tar.gz | tar -xz \ +&& cd mykrobe-${mykrobe_version} \ +&& pip3 install requests \ +&& pip3 install . \ +&& ln -s /usr/local/bin/mccortex/bin/mccortex31 /usr/local/lib/python3.8/dist-packages/mykrobe/cortex/mccortex31 \ +&& mykrobe panels update_metadata \ +&& mykrobe panels update_species all \ +&& cd .. + +RUN curl -fsSL https://github.com/lh3/bwa/archive/v${bwa_version}.tar.gz | tar -C /usr/local/bin -xz \ +&& make -C /usr/local/bin/bwa-${bwa_version} \ +&& chmod +x /usr/local/bin/bwa-${bwa_version}/bwa + +RUN unset DEBIAN_FRONTEND + +ENV LC_ALL en_US.UTF-8 \ +LANG en_US.UTF-8 \ +LANGUAGE en_US.UTF-8 + diff --git a/test_scripts/docker/Dockerfile.tbprofiler-0.9.8 b/test_scripts/docker/Dockerfile.tbprofiler-0.9.8 new file mode 100644 index 0000000..686c9c4 --- /dev/null +++ b/test_scripts/docker/Dockerfile.tbprofiler-0.9.8 @@ -0,0 +1,54 @@ +FROM mambaorg/micromamba:1.3.0 as app + +#copy the reference genome to pre-compute our index +COPY resources/tuberculosis.fasta /data/tuberculosis.fasta + +USER root +WORKDIR / + +ARG TBPROFILER_VER="5.0.1" + +# this version is the shortened commit hash on the `master` branch here https://github.com/jodyphelan/tbdb/ +# commits are found on https://github.com/jodyphelan/tbdb/commits/master +# this was the latest commit as of 2023-10-26 +ARG TBDB_VER="e25540b" + +# LABEL instructions tag the image with metadata that might be important to the user +LABEL base.image="micromamba:1.3.0" +LABEL dockerfile.version="1" +LABEL software="tbprofiler" +LABEL software.version="${TBPROFILER_VER}" +LABEL description="The pipeline aligns reads to the H37Rv reference using bowtie2, BWA or minimap2 and then calls variants using bcftools. These variants are then compared to a drug-resistance database." +LABEL website="https://github.com/jodyphelan/TBProfiler/" +LABEL license="https://github.com/jodyphelan/TBProfiler/blob/master/LICENSE" +LABEL maintainer="John Arnn" +LABEL maintainer.email="jarnn@utah.gov" +LABEL maintainer2="Curtis Kapsak" +LABEL maintainer2.email="kapsakcj@gmail.com" + +# Install dependencies via apt-get; cleanup apt garbage +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget \ + ca-certificates \ + procps && \ + apt-get autoclean && rm -rf /var/lib/apt/lists/* + +# install tb-profiler via bioconda; install into 'base' conda env +RUN micromamba install --yes --name base --channel conda-forge --channel bioconda \ + tb-profiler=${TBPROFILER_VER} + +RUN micromamba install --yes --name base --channel conda-forge --channel bioconda gatk4 +RUN micromamba install --yes --name base --channel conda-forge --channel bioconda samtools +RUN micromamba install --yes --name base --channel conda-forge jq +RUN micromamba clean --all --yes + +# hardcode 'base' env bin into PATH, so conda env does not have to be "activated" at run time +ENV PATH="/opt/conda/bin:${PATH}" + +# Version of database can be confirmed at /opt/conda/share/tbprofiler/tbdb.version.json +# can also run 'tb-profiler list_db' to find the same version info +# In 5.0.1 updating_tbdb does not work with tb-profiler update_tbdb --commit ${TBDB_VER} +RUN tb-profiler update_tbdb --commit ${TBDB_VER} + +WORKDIR /data +RUN tb-profiler update_tbdb --match_ref tuberculosis.fasta diff --git a/test_scripts/docker/Dockerfile.vcfpredict-0.9.8 b/test_scripts/docker/Dockerfile.vcfpredict-0.9.8 new file mode 100644 index 0000000..9de1bb7 --- /dev/null +++ b/test_scripts/docker/Dockerfile.vcfpredict-0.9.8 @@ -0,0 +1,27 @@ +FROM ubuntu:20.04 + +LABEL maintainer="pricea35@cardiff.ac.uk" \ +about.summary="container for the vcf predict workflow" + +#add run-vcf to container +COPY bin/ /opt/bin/ +ENV PATH=/opt/bin:$PATH + +ENV PACKAGES="procps curl wget git build-essential libhdf5-dev libffi-dev r-base-core jq" \ +PYTHON="python3 python3-pip python3-dev" + +ENV vcfmix_version=d4693344bf612780723e39ce27c8ae3868f95417 + +#apt updates +RUN apt-get update \ +&& DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata \ +&& apt-get install -y $PACKAGES $PYTHON \ +&& apt-get install -y python3-packaging \ +&& git clone https://github.com/JeremyWesthead/VCFMIX.git \ +&& cd VCFMIX \ +&& git checkout ${vcfmix_version} \ +&& pip3 install recursive_diff \ +&& pip3 install awscli \ +&& pip3 install . \ +&& cp -r data /usr/local/lib/python3.8/dist-packages \ +&& cd .. diff --git a/test_scripts/docker/docker/Dockerfile.clockwork-0.9.8 b/test_scripts/docker/docker/Dockerfile.clockwork-0.9.8 new file mode 100644 index 0000000..d45b6c4 --- /dev/null +++ b/test_scripts/docker/docker/Dockerfile.clockwork-0.9.8 @@ -0,0 +1,120 @@ +FROM debian:buster + +LABEL maintainer="pricea35@cardiff.ac.uk" \ +about.summary="container for the clockwork workflow" + +ENV samtools_version=1.12 \ +htslib_version=1.12 \ +bcftools_version=1.12 \ +minimap2_version=2.17 \ +picard_version=2.18.16 \ +gramtools_version=8af53f6c8c0d72ef95223e89ab82119b717044f2 \ +vt_version=2187ff6347086e38f71bd9f8ca622cd7dcfbb40c \ +minos_version=0.11.0 \ +cortex_version=3a235272e4e0121be64527f01e73f9e066d378d3 \ +vcftools_version=0.1.15 \ +mccortex_version=97aba198d632ee98ac1aa496db33d1a7a8cb7e51 \ +stampy_version=1.0.32r3761 \ +python_version=3.6.5 \ +clockwork_version=2364dec4cbf25c844575e19e8fe0a319d10721b5 + +ENV PACKAGES="procps curl git build-essential wget zlib1g-dev pkg-config jq r-base-core rsync autoconf libncurses-dev libbz2-dev liblzma-dev libcurl4-openssl-dev cmake tabix libvcflib-tools libssl-dev software-properties-common perl locales locales-all" \ +PYTHON="python2.7 python-dev" + +COPY bin/ /opt/bin/ +ENV PATH=/opt/bin:$PATH + + +RUN apt-get update \ +&& apt-get install -y $PACKAGES $PYTHON \ +&& curl -fsSL https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tgz | tar -xz \ +&& cd Python-${python_version} \ +&& ./configure --enable-optimizations \ +&& make altinstall \ +&& cd .. \ +&& ln -s /usr/local/bin/python3.6 /usr/local/bin/python3 \ +&& ln -s /usr/local/bin/pip3.6 /usr/local/bin/pip3 \ +&& pip3 install --upgrade pip \ +&& pip3 install 'cluster_vcf_records==0.13.1' pysam setuptools awscli \ +&& wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - \ +&& add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ \ +&& apt-get update && apt-get install -y adoptopenjdk-8-hotspot + + +RUN curl -fsSL https://github.com/samtools/samtools/archive/${samtools_version}.tar.gz | tar -xz \ +&& curl -fsSL https://github.com/samtools/htslib/releases/download/${htslib_version}/htslib-${htslib_version}.tar.bz2 | tar -xj \ +&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} \ +&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ +&& rm -r samtools-${samtools_version} \ +&& curl -fsSL https://github.com/samtools/bcftools/archive/refs/tags/${bcftools_version}.tar.gz | tar -xz \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ +&& rm -r bcftools-${bcftools_version} + + +RUN curl -fsSL minimap2-${minimap2_version}.tar.gz https://github.com/lh3/minimap2/archive/v${minimap2_version}.tar.gz | tar -xz \ +&& cd minimap2-${minimap2_version} \ +&& make \ +&& chmod +x minimap2 \ +&& mv minimap2 /usr/local/bin \ +&& cd .. \ +&& rm -r minimap2-${minimap2_version} \ +&& wget https://github.com/broadinstitute/picard/releases/download/${picard_version}/picard.jar -O /usr/local/bin/picard.jar + + +RUN git clone https://github.com/atks/vt.git vt-git \ +&& cd vt-git \ +&& git checkout ${vt_version} \ +&& make \ +&& cd .. \ +&& mv vt-git/vt /usr/local/bin \ +&& pip3 install tox "six>=1.14.0" \ +&& git clone https://github.com/iqbal-lab-org/gramtools \ +&& cd gramtools \ +&& git checkout ${gramtools_version} \ +&& pip3 install . \ +&& cd .. \ +&& pip3 install cython \ +&& pip3 install git+https://github.com/iqbal-lab-org/minos@v${minos_version} + + +RUN git clone --recursive https://github.com/iqbal-lab/cortex.git \ +&& cd cortex \ +&& git checkout ${cortex_version} \ +&& bash install.sh \ +&& make NUM_COLS=1 cortex_var \ +&& make NUM_COLS=2 cortex_var \ +&& cd .. \ +&& mkdir bioinf-tools \ +&& cd bioinf-tools \ +&& curl -fsSL http://www.well.ox.ac.uk/~gerton/software/Stampy/stampy-${stampy_version}.tgz | tar -xz \ +&& make -C stampy-* \ +&& cp -s stampy-*/stampy.py . \ +&& curl -fsSL https://github.com/vcftools/vcftools/releases/download/v${vcftools_version}/vcftools-${vcftools_version}.tar.gz | tar -xz \ +&& cd vcftools-${vcftools_version} \ +&& ./configure --prefix $PWD/install \ +&& make && make install \ +&& ln -s src/perl/ . \ +&& cd .. \ +&& git clone --recursive https://github.com/mcveanlab/mccortex \ +&& cd mccortex \ +&& git checkout ${mccortex_version} \ +&& make all \ +&& cd .. \ +&& cp -s mccortex/bin/mccortex31 . \ +&& cd .. \ +&& git clone https://github.com/iqbal-lab-org/clockwork \ +&& cd clockwork \ +&& git checkout ${clockwork_version} \ +&& cd python \ +&& pip3 install . \ +&& chmod +x scripts/clockwork + +ENV CLOCKWORK_CORTEX_DIR=/cortex \ +PATH=${PATH}:/clockwork/python/scripts \ +PICARD_JAR=/usr/local/bin/picard.jar + +ENV LC_ALL en_US.UTF-8 \ +LANG en_US.UTF-8 \ +LANGUAGE en_US.UTF-8 + diff --git a/test_scripts/docker/docker/Dockerfile.preprocessing-0.9.8 b/test_scripts/docker/docker/Dockerfile.preprocessing-0.9.8 new file mode 100644 index 0000000..1905969 --- /dev/null +++ b/test_scripts/docker/docker/Dockerfile.preprocessing-0.9.8 @@ -0,0 +1,131 @@ +FROM ubuntu:focal + +LABEL maintainer="pricea35@cardiff.ac.uk" \ +about.summary="container for the preprocessing workflow" + +ENV samtools_version=1.12 \ +bcftools_version=1.12 \ +htslib_version=1.12 \ +bedtools_version=2.29.2 \ +bowtie2_version=2.4.2 \ +fastp_version=0.20.1 \ +fastqc_version=0.11.9 \ +fqtools_version=2.3 \ +kraken2_version=2.1.1 \ +afanc_version=0.10.2 \ +mykrobe_version=0.12.1 \ +bwa_version=0.7.17 \ +mash_version=2.3 \ +fastani_version=1.33 + +ENV PACKAGES="procps curl git wget build-essential zlib1g-dev libncurses-dev libz-dev libbz2-dev liblzma-dev libcurl4-openssl-dev libgsl-dev rsync unzip ncbi-blast+ pigz jq libtbb-dev openjdk-11-jre-headless autoconf r-base-core locales locales-all" \ +PYTHON="python3 python3-pip python3-dev" \ +PYTHON_PACKAGES="biopython awscli boto3" + +ENV PATH=${PATH}:/usr/local/bin/mccortex/bin:/usr/local/bin/bwa-${bwa_version}:/opt/edirect \ +LD_LIBRARY_PATH=/usr/local/lib + +RUN export DEBIAN_FRONTEND="noninteractive" + +COPY bin/ /opt/bin/ +ENV PATH=/opt/bin:$PATH + +RUN apt-get update \ +&& DEBIAN_FRONTEND="noninteractive" apt-get install -y $PACKAGES $PYTHON \ +&& pip3 install --upgrade pip \ +&& pip3 install $PYTHON_PACKAGES \ +&& ln -s /usr/bin/python3 /usr/bin/python + +RUN curl -fsSL https://github.com/samtools/samtools/archive/${samtools_version}.tar.gz | tar -xz \ +&& curl -fsSL https://github.com/samtools/htslib/releases/download/${htslib_version}/htslib-${htslib_version}.tar.bz2 | tar -xj \ +&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} \ +&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ +&& rm -r samtools-${samtools_version} \ +&& curl -fsSL https://github.com/samtools/bcftools/archive/refs/tags/${bcftools_version}.tar.gz | tar -xz \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ +&& rm -r bcftools-${bcftools_version} + +RUN curl -fsSL https://github.com/alastair-droop/fqtools/archive/v${fqtools_version}.tar.gz | tar -xz \ +&& mv htslib-${htslib_version} fqtools-${fqtools_version} \ +&& cd fqtools-${fqtools_version} \ +&& mv htslib-${htslib_version} htslib \ +&& cd htslib \ +&& autoreconf -i \ +&& ./configure \ +&& make \ +&& make install \ +&& cd .. \ +&& make \ +&& mv bin/* /usr/local/bin \ +&& chmod +x /usr/local/bin/fqtools \ +&& cd .. \ +&& rm -r fqtools-${fqtools_version} + +RUN curl -fsSL https://github.com/arq5x/bedtools2/releases/download/v${bedtools_version}/bedtools-${bedtools_version}.tar.gz | tar -xz \ +&& make -C bedtools2 \ +&& mv bedtools2/bin/* /usr/local/bin \ +&& rm -r bedtools2 + +RUN curl -fsSL https://sourceforge.net/projects/bowtie-bio/files/bowtie2/${bowtie2_version}/bowtie2-${bowtie2_version}-source.zip -o bowtie2-${bowtie2_version}-source.zip \ +&& unzip bowtie2-${bowtie2_version}-source.zip \ +&& make -C bowtie2-${bowtie2_version} prefix=/usr/local install \ +&& rm -r bowtie2-${bowtie2_version} \ +&& rm bowtie2-${bowtie2_version}-source.zip + +RUN curl -fsSL https://github.com/OpenGene/fastp/archive/v${fastp_version}.tar.gz | tar -xz \ +&& cd fastp-${fastp_version} \ +&& make \ +&& make install \ +&& cd .. \ +&& rm -r fastp-${fastp_version} + +RUN wget https://www.bioinformatics.babraham.ac.uk/projects/fastqc/fastqc_v${fastqc_version}.zip \ +&& unzip fastqc_v${fastqc_version}.zip \ +&& chmod +x FastQC/fastqc \ +&& mv FastQC/* /usr/local/bin \ +&& rm fastqc_v${fastqc_version}.zip \ +&& rm -r FastQC + +RUN curl -fsSL https://github.com/DerrickWood/kraken2/archive/v${kraken2_version}.tar.gz | tar -xz \ +&& cd kraken2-${kraken2_version} \ +&& ./install_kraken2.sh /usr/local/bin \ +&& cd .. + +RUN curl -fsSL https://github.com/ArthurVM/Afanc/archive/refs/tags/v${afanc_version}-alpha.tar.gz | tar -xz \ +&& cd Afanc-${afanc_version}-alpha \ +&& pip3 install ./ \ +&& cd .. \ +&& curl -fsSL "https://github.com/marbl/Mash/releases/download/v${mash_version}/mash-Linux64-v${mash_version}.tar" | tar -x \ +&& mv mash-Linux64-v${mash_version}/mash /usr/local/bin \ +&& rm -r mash-Linux* \ +&& wget https://github.com/ParBLiSS/FastANI/releases/download/v${fastani_version}/fastANI-Linux64-v${fastani_version}.zip \ +&& unzip fastANI-Linux64-v${fastani_version}.zip \ +&& mv fastANI /usr/local/bin + +RUN sh -c "$(curl -fsSL ftp://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/install-edirect.sh)" \ +&& mkdir -p /opt/edirect \ +&& mv /root/edirect/* /opt/edirect + +RUN git clone --recursive -b geno_kmer_count https://github.com/phelimb/mccortex \ +&& make -C mccortex \ +&& mv mccortex /usr/local/bin \ +&& curl -fsSL mykrobe-${mykrobe_version}.tar.gz https://github.com/Mykrobe-tools/mykrobe/archive/v${mykrobe_version}.tar.gz | tar -xz \ +&& cd mykrobe-${mykrobe_version} \ +&& pip3 install requests \ +&& pip3 install . \ +&& ln -s /usr/local/bin/mccortex/bin/mccortex31 /usr/local/lib/python3.8/dist-packages/mykrobe/cortex/mccortex31 \ +&& mykrobe panels update_metadata \ +&& mykrobe panels update_species all \ +&& cd .. + +RUN curl -fsSL https://github.com/lh3/bwa/archive/v${bwa_version}.tar.gz | tar -C /usr/local/bin -xz \ +&& make -C /usr/local/bin/bwa-${bwa_version} \ +&& chmod +x /usr/local/bin/bwa-${bwa_version}/bwa + +RUN unset DEBIAN_FRONTEND + +ENV LC_ALL en_US.UTF-8 \ +LANG en_US.UTF-8 \ +LANGUAGE en_US.UTF-8 + diff --git a/test_scripts/docker/docker/Dockerfile.tbprofiler-0.9.8 b/test_scripts/docker/docker/Dockerfile.tbprofiler-0.9.8 new file mode 100644 index 0000000..686c9c4 --- /dev/null +++ b/test_scripts/docker/docker/Dockerfile.tbprofiler-0.9.8 @@ -0,0 +1,54 @@ +FROM mambaorg/micromamba:1.3.0 as app + +#copy the reference genome to pre-compute our index +COPY resources/tuberculosis.fasta /data/tuberculosis.fasta + +USER root +WORKDIR / + +ARG TBPROFILER_VER="5.0.1" + +# this version is the shortened commit hash on the `master` branch here https://github.com/jodyphelan/tbdb/ +# commits are found on https://github.com/jodyphelan/tbdb/commits/master +# this was the latest commit as of 2023-10-26 +ARG TBDB_VER="e25540b" + +# LABEL instructions tag the image with metadata that might be important to the user +LABEL base.image="micromamba:1.3.0" +LABEL dockerfile.version="1" +LABEL software="tbprofiler" +LABEL software.version="${TBPROFILER_VER}" +LABEL description="The pipeline aligns reads to the H37Rv reference using bowtie2, BWA or minimap2 and then calls variants using bcftools. These variants are then compared to a drug-resistance database." +LABEL website="https://github.com/jodyphelan/TBProfiler/" +LABEL license="https://github.com/jodyphelan/TBProfiler/blob/master/LICENSE" +LABEL maintainer="John Arnn" +LABEL maintainer.email="jarnn@utah.gov" +LABEL maintainer2="Curtis Kapsak" +LABEL maintainer2.email="kapsakcj@gmail.com" + +# Install dependencies via apt-get; cleanup apt garbage +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget \ + ca-certificates \ + procps && \ + apt-get autoclean && rm -rf /var/lib/apt/lists/* + +# install tb-profiler via bioconda; install into 'base' conda env +RUN micromamba install --yes --name base --channel conda-forge --channel bioconda \ + tb-profiler=${TBPROFILER_VER} + +RUN micromamba install --yes --name base --channel conda-forge --channel bioconda gatk4 +RUN micromamba install --yes --name base --channel conda-forge --channel bioconda samtools +RUN micromamba install --yes --name base --channel conda-forge jq +RUN micromamba clean --all --yes + +# hardcode 'base' env bin into PATH, so conda env does not have to be "activated" at run time +ENV PATH="/opt/conda/bin:${PATH}" + +# Version of database can be confirmed at /opt/conda/share/tbprofiler/tbdb.version.json +# can also run 'tb-profiler list_db' to find the same version info +# In 5.0.1 updating_tbdb does not work with tb-profiler update_tbdb --commit ${TBDB_VER} +RUN tb-profiler update_tbdb --commit ${TBDB_VER} + +WORKDIR /data +RUN tb-profiler update_tbdb --match_ref tuberculosis.fasta diff --git a/test_scripts/docker/docker/Dockerfile.vcfpredict-0.9.8 b/test_scripts/docker/docker/Dockerfile.vcfpredict-0.9.8 new file mode 100644 index 0000000..9de1bb7 --- /dev/null +++ b/test_scripts/docker/docker/Dockerfile.vcfpredict-0.9.8 @@ -0,0 +1,27 @@ +FROM ubuntu:20.04 + +LABEL maintainer="pricea35@cardiff.ac.uk" \ +about.summary="container for the vcf predict workflow" + +#add run-vcf to container +COPY bin/ /opt/bin/ +ENV PATH=/opt/bin:$PATH + +ENV PACKAGES="procps curl wget git build-essential libhdf5-dev libffi-dev r-base-core jq" \ +PYTHON="python3 python3-pip python3-dev" + +ENV vcfmix_version=d4693344bf612780723e39ce27c8ae3868f95417 + +#apt updates +RUN apt-get update \ +&& DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata \ +&& apt-get install -y $PACKAGES $PYTHON \ +&& apt-get install -y python3-packaging \ +&& git clone https://github.com/JeremyWesthead/VCFMIX.git \ +&& cd VCFMIX \ +&& git checkout ${vcfmix_version} \ +&& pip3 install recursive_diff \ +&& pip3 install awscli \ +&& pip3 install . \ +&& cp -r data /usr/local/lib/python3.8/dist-packages \ +&& cd .. diff --git a/test_scripts/mainscripts/TM01_main.nf b/test_scripts/mainscripts/TM01_main.nf index f8bd578..95f0e7f 100644 --- a/test_scripts/mainscripts/TM01_main.nf +++ b/test_scripts/mainscripts/TM01_main.nf @@ -5,7 +5,6 @@ nextflow.enable.dsl=2 // import subworkflows include {tm01} from './../workflows/TM01.nf' -include {clockwork} from './../../tb-pipeline/workflows/clockwork.nf' /* ANSI escape codes to allow colour-coded output messages @@ -24,12 +23,7 @@ if (params.help) { def helpMessage() { log.info """ ======================================================================== -M Y C O B A C T E R I A L P I P E L I N E - -Cleans and QCs reads with fastp and FastQC, classifies with Kraken2 & Mykrobe, removes non-bacterial content, and - by alignment to any minority genomes - disambiguates mixtures of bacterial reads. - -Takes as input one directory containing pairs of fastq(.gz) or bam files. -Produces as output one directory per sample, containing the relevant reports & a pair of cleaned fastqs. +TM01 TEST PIPELINE Mandatory and conditional parameters: ------------------------------------------------------------------------ @@ -38,29 +32,7 @@ Mandatory and conditional parameters: have to literally take the form "*.fastq"; see --pattern --pattern regex to match files in input_dir, e.g. "*_R{1,2}.fq.gz". Only mandatory if --filetype is "fastq" --output_dir output directory, in which will be created subdirectories matching base name of fastq/bam files ---unmix_myco do you want to disambiguate mixed-mycobacterial samples by read alignment? One of "yes" or "no" - if "yes" workflow will remove reads mapping to any minority mycobacterial genomes but in doing so - WILL ALMOST CERTAINLY ALSO reduce coverage of the principal species - if "no" then mixed-mycobacterial samples will be left alone. Mixtures of mycobacteria + non-mycobacteria - will still be disambiguated ---kraken_db directory containing Kraken2 database files (obtain from https://benlangmead.github.io/aws-indexes/k2) ---bowtie2_index directory containing Bowtie2 index (obtain from ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/hg19_1kgmaj_bt2.zip - This is the Langmead lab pre-built major-allele-SNP reference; see https://github.com/BenLangmead/bowtie-majref) ---bowtie_index_name name of the bowtie index, e.g. hg19_1kgmaj - -Optional parameters: ------------------------------------------------------------------------- ---species principal species in each sample, assuming genus Mycobacterium - if parameter used, takes 1 of 10 values: abscessus, africanum, avium, bovis, chelonae, chimaera, - fortuitum, intracellulare, kansasii, tuberculosis - default: null - using this parameter will apply an additional sanity test to your sample - - if you DO NOT use this parameter (default option), pipeline will determine principal species from - the reads and consider any other species a contaminant - if you DO use this parameter, pipeline will expect this to be the principal species. It will fail - the sample if reads from this species are not actually the majority Profiles: @@ -98,12 +70,7 @@ if ( params.output_dir == "" ) { if ( ( params.filetype != "fastq" ) && ( params.filetype != "bam" ) ) { exit 1, "error: --filetype is mandatory and must be either \"fastq\" or \"bam\"" } -if ( ( params.unmix_myco != "yes" ) && ( params.unmix_myco != "no" ) ) { - exit 1, "error: --unmix_myco is mandatory and must be either \"yes\" or \"no\"" -} -if ( ( params.species != "null" ) && ( params.species != "abscessus" ) && ( params.species != "africanum" ) && ( params.species != "avium" ) && ( params.species != "bovis" ) && ( params.species != "chelonae" ) && ( params.species != "chimaera" ) && ( params.species != "fortuitum" ) && ( params.species != "intracellulare" ) && ( params.species != "kansasii" ) && ( params.species != "tuberculosis" ) ) { - exit 1, "error: --species is optional, but if used should be one of either abscessus, africanum, avium, bovis, chelonae, chimaera, fortuitum, intracellulare, kansasii, tuberculosis" -} + log.info """ ======================================================================== @@ -115,11 +82,6 @@ Parameters used: --filetype ${params.filetype} --pattern ${params.pattern} --output_dir ${params.output_dir} ---unmix_myco ${params.unmix_myco} ---kraken_db ${params.kraken_db} ---bowtie2_index ${params.bowtie2_index} ---bowtie_index_name ${params.bowtie_index_name} ---species ${params.species} Runtime data: ------------------------------------------------------------------------ @@ -148,21 +110,18 @@ workflow { if ( params.filetype == "fastq" ) { pattern = params.pattern - reads = indir + pattern - numfiles = file(reads) // count the number of files + reads = indir + pattern + numfiles = file(reads) // count the number of files Channel.fromFilePairs(reads, flat: true, checkIfExists: true, size: -1) .ifEmpty { error "cannot find any reads matching ${pattern} in ${indir}" } .set{ input_files } } - // create channels for kraken2 database and bowtie2 index - krakenDB = Channel.fromPath( "${params.kraken_db}/*.k2d" ) - bowtie_dir = Channel.fromPath( "${params.bowtie2_index}/*.bt2" ) // call preprocressing subworkflow main: - tm01(input_files, krakenDB, bowtie_dir) + tm01(input_files) } workflow.onComplete { diff --git a/test_scripts/mainscripts/config/config/containers.config b/test_scripts/mainscripts/config/config/containers.config new file mode 100644 index 0000000..e961b71 --- /dev/null +++ b/test_scripts/mainscripts/config/config/containers.config @@ -0,0 +1,48 @@ +params{ + container_enabled = "true" + container_enabled = "true" +} + + +process { + update_tbprofiler = "false" + + + withLabel:low_cpu {cpus = 2} + withLabel:normal_cpu { cpus = 8 } + withLabel:low_memory { memory = '5GB' } + withLabel:medium_memory { memory = '10GB' } + withLabel:high_memory { memory = '18GB' } + + withLabel:getversion { + container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.8" + } + + withLabel:preprocessing { + container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.8" + } + + withLabel:tbprofiler { + container = "quay.io/pathogen-genomics-cymru/tbprofiler:0.9.8" + } + + withName:downloadContamGenomes { + shell = ['/bin/bash','-u'] + errorStrategy = { task.exitStatus in 100..113 ? 'retry' : 'terminate' } + maxRetries = 5 + } + + withLabel:retryAfanc { + shell = ['/bin/bash','-u'] + errorStrategy = {task.exitStatus == 1 ? 'retry' : 'ignore' } + maxRetries = 5 + } + + withLabel:clockwork { + container = "quay.io/pathogen-genomics-cymru/clockwork:0.9.8" + } + + withLabel:vcfpredict { + container = "quay.io/pathogen-genomics-cymru/vcfpredict:0.9.8" + } + } \ No newline at end of file diff --git a/test_scripts/mainscripts/config/containers.config b/test_scripts/mainscripts/config/containers.config new file mode 100644 index 0000000..e961b71 --- /dev/null +++ b/test_scripts/mainscripts/config/containers.config @@ -0,0 +1,48 @@ +params{ + container_enabled = "true" + container_enabled = "true" +} + + +process { + update_tbprofiler = "false" + + + withLabel:low_cpu {cpus = 2} + withLabel:normal_cpu { cpus = 8 } + withLabel:low_memory { memory = '5GB' } + withLabel:medium_memory { memory = '10GB' } + withLabel:high_memory { memory = '18GB' } + + withLabel:getversion { + container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.8" + } + + withLabel:preprocessing { + container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.8" + } + + withLabel:tbprofiler { + container = "quay.io/pathogen-genomics-cymru/tbprofiler:0.9.8" + } + + withName:downloadContamGenomes { + shell = ['/bin/bash','-u'] + errorStrategy = { task.exitStatus in 100..113 ? 'retry' : 'terminate' } + maxRetries = 5 + } + + withLabel:retryAfanc { + shell = ['/bin/bash','-u'] + errorStrategy = {task.exitStatus == 1 ? 'retry' : 'ignore' } + maxRetries = 5 + } + + withLabel:clockwork { + container = "quay.io/pathogen-genomics-cymru/clockwork:0.9.8" + } + + withLabel:vcfpredict { + container = "quay.io/pathogen-genomics-cymru/vcfpredict:0.9.8" + } + } \ No newline at end of file diff --git a/test_scripts/mainscripts/docker/Dockerfile.clockwork-0.9.8 b/test_scripts/mainscripts/docker/Dockerfile.clockwork-0.9.8 new file mode 100644 index 0000000..d45b6c4 --- /dev/null +++ b/test_scripts/mainscripts/docker/Dockerfile.clockwork-0.9.8 @@ -0,0 +1,120 @@ +FROM debian:buster + +LABEL maintainer="pricea35@cardiff.ac.uk" \ +about.summary="container for the clockwork workflow" + +ENV samtools_version=1.12 \ +htslib_version=1.12 \ +bcftools_version=1.12 \ +minimap2_version=2.17 \ +picard_version=2.18.16 \ +gramtools_version=8af53f6c8c0d72ef95223e89ab82119b717044f2 \ +vt_version=2187ff6347086e38f71bd9f8ca622cd7dcfbb40c \ +minos_version=0.11.0 \ +cortex_version=3a235272e4e0121be64527f01e73f9e066d378d3 \ +vcftools_version=0.1.15 \ +mccortex_version=97aba198d632ee98ac1aa496db33d1a7a8cb7e51 \ +stampy_version=1.0.32r3761 \ +python_version=3.6.5 \ +clockwork_version=2364dec4cbf25c844575e19e8fe0a319d10721b5 + +ENV PACKAGES="procps curl git build-essential wget zlib1g-dev pkg-config jq r-base-core rsync autoconf libncurses-dev libbz2-dev liblzma-dev libcurl4-openssl-dev cmake tabix libvcflib-tools libssl-dev software-properties-common perl locales locales-all" \ +PYTHON="python2.7 python-dev" + +COPY bin/ /opt/bin/ +ENV PATH=/opt/bin:$PATH + + +RUN apt-get update \ +&& apt-get install -y $PACKAGES $PYTHON \ +&& curl -fsSL https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tgz | tar -xz \ +&& cd Python-${python_version} \ +&& ./configure --enable-optimizations \ +&& make altinstall \ +&& cd .. \ +&& ln -s /usr/local/bin/python3.6 /usr/local/bin/python3 \ +&& ln -s /usr/local/bin/pip3.6 /usr/local/bin/pip3 \ +&& pip3 install --upgrade pip \ +&& pip3 install 'cluster_vcf_records==0.13.1' pysam setuptools awscli \ +&& wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - \ +&& add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ \ +&& apt-get update && apt-get install -y adoptopenjdk-8-hotspot + + +RUN curl -fsSL https://github.com/samtools/samtools/archive/${samtools_version}.tar.gz | tar -xz \ +&& curl -fsSL https://github.com/samtools/htslib/releases/download/${htslib_version}/htslib-${htslib_version}.tar.bz2 | tar -xj \ +&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} \ +&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ +&& rm -r samtools-${samtools_version} \ +&& curl -fsSL https://github.com/samtools/bcftools/archive/refs/tags/${bcftools_version}.tar.gz | tar -xz \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ +&& rm -r bcftools-${bcftools_version} + + +RUN curl -fsSL minimap2-${minimap2_version}.tar.gz https://github.com/lh3/minimap2/archive/v${minimap2_version}.tar.gz | tar -xz \ +&& cd minimap2-${minimap2_version} \ +&& make \ +&& chmod +x minimap2 \ +&& mv minimap2 /usr/local/bin \ +&& cd .. \ +&& rm -r minimap2-${minimap2_version} \ +&& wget https://github.com/broadinstitute/picard/releases/download/${picard_version}/picard.jar -O /usr/local/bin/picard.jar + + +RUN git clone https://github.com/atks/vt.git vt-git \ +&& cd vt-git \ +&& git checkout ${vt_version} \ +&& make \ +&& cd .. \ +&& mv vt-git/vt /usr/local/bin \ +&& pip3 install tox "six>=1.14.0" \ +&& git clone https://github.com/iqbal-lab-org/gramtools \ +&& cd gramtools \ +&& git checkout ${gramtools_version} \ +&& pip3 install . \ +&& cd .. \ +&& pip3 install cython \ +&& pip3 install git+https://github.com/iqbal-lab-org/minos@v${minos_version} + + +RUN git clone --recursive https://github.com/iqbal-lab/cortex.git \ +&& cd cortex \ +&& git checkout ${cortex_version} \ +&& bash install.sh \ +&& make NUM_COLS=1 cortex_var \ +&& make NUM_COLS=2 cortex_var \ +&& cd .. \ +&& mkdir bioinf-tools \ +&& cd bioinf-tools \ +&& curl -fsSL http://www.well.ox.ac.uk/~gerton/software/Stampy/stampy-${stampy_version}.tgz | tar -xz \ +&& make -C stampy-* \ +&& cp -s stampy-*/stampy.py . \ +&& curl -fsSL https://github.com/vcftools/vcftools/releases/download/v${vcftools_version}/vcftools-${vcftools_version}.tar.gz | tar -xz \ +&& cd vcftools-${vcftools_version} \ +&& ./configure --prefix $PWD/install \ +&& make && make install \ +&& ln -s src/perl/ . \ +&& cd .. \ +&& git clone --recursive https://github.com/mcveanlab/mccortex \ +&& cd mccortex \ +&& git checkout ${mccortex_version} \ +&& make all \ +&& cd .. \ +&& cp -s mccortex/bin/mccortex31 . \ +&& cd .. \ +&& git clone https://github.com/iqbal-lab-org/clockwork \ +&& cd clockwork \ +&& git checkout ${clockwork_version} \ +&& cd python \ +&& pip3 install . \ +&& chmod +x scripts/clockwork + +ENV CLOCKWORK_CORTEX_DIR=/cortex \ +PATH=${PATH}:/clockwork/python/scripts \ +PICARD_JAR=/usr/local/bin/picard.jar + +ENV LC_ALL en_US.UTF-8 \ +LANG en_US.UTF-8 \ +LANGUAGE en_US.UTF-8 + diff --git a/test_scripts/mainscripts/docker/Dockerfile.preprocessing-0.9.8 b/test_scripts/mainscripts/docker/Dockerfile.preprocessing-0.9.8 new file mode 100644 index 0000000..1905969 --- /dev/null +++ b/test_scripts/mainscripts/docker/Dockerfile.preprocessing-0.9.8 @@ -0,0 +1,131 @@ +FROM ubuntu:focal + +LABEL maintainer="pricea35@cardiff.ac.uk" \ +about.summary="container for the preprocessing workflow" + +ENV samtools_version=1.12 \ +bcftools_version=1.12 \ +htslib_version=1.12 \ +bedtools_version=2.29.2 \ +bowtie2_version=2.4.2 \ +fastp_version=0.20.1 \ +fastqc_version=0.11.9 \ +fqtools_version=2.3 \ +kraken2_version=2.1.1 \ +afanc_version=0.10.2 \ +mykrobe_version=0.12.1 \ +bwa_version=0.7.17 \ +mash_version=2.3 \ +fastani_version=1.33 + +ENV PACKAGES="procps curl git wget build-essential zlib1g-dev libncurses-dev libz-dev libbz2-dev liblzma-dev libcurl4-openssl-dev libgsl-dev rsync unzip ncbi-blast+ pigz jq libtbb-dev openjdk-11-jre-headless autoconf r-base-core locales locales-all" \ +PYTHON="python3 python3-pip python3-dev" \ +PYTHON_PACKAGES="biopython awscli boto3" + +ENV PATH=${PATH}:/usr/local/bin/mccortex/bin:/usr/local/bin/bwa-${bwa_version}:/opt/edirect \ +LD_LIBRARY_PATH=/usr/local/lib + +RUN export DEBIAN_FRONTEND="noninteractive" + +COPY bin/ /opt/bin/ +ENV PATH=/opt/bin:$PATH + +RUN apt-get update \ +&& DEBIAN_FRONTEND="noninteractive" apt-get install -y $PACKAGES $PYTHON \ +&& pip3 install --upgrade pip \ +&& pip3 install $PYTHON_PACKAGES \ +&& ln -s /usr/bin/python3 /usr/bin/python + +RUN curl -fsSL https://github.com/samtools/samtools/archive/${samtools_version}.tar.gz | tar -xz \ +&& curl -fsSL https://github.com/samtools/htslib/releases/download/${htslib_version}/htslib-${htslib_version}.tar.bz2 | tar -xj \ +&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} \ +&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ +&& rm -r samtools-${samtools_version} \ +&& curl -fsSL https://github.com/samtools/bcftools/archive/refs/tags/${bcftools_version}.tar.gz | tar -xz \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ +&& rm -r bcftools-${bcftools_version} + +RUN curl -fsSL https://github.com/alastair-droop/fqtools/archive/v${fqtools_version}.tar.gz | tar -xz \ +&& mv htslib-${htslib_version} fqtools-${fqtools_version} \ +&& cd fqtools-${fqtools_version} \ +&& mv htslib-${htslib_version} htslib \ +&& cd htslib \ +&& autoreconf -i \ +&& ./configure \ +&& make \ +&& make install \ +&& cd .. \ +&& make \ +&& mv bin/* /usr/local/bin \ +&& chmod +x /usr/local/bin/fqtools \ +&& cd .. \ +&& rm -r fqtools-${fqtools_version} + +RUN curl -fsSL https://github.com/arq5x/bedtools2/releases/download/v${bedtools_version}/bedtools-${bedtools_version}.tar.gz | tar -xz \ +&& make -C bedtools2 \ +&& mv bedtools2/bin/* /usr/local/bin \ +&& rm -r bedtools2 + +RUN curl -fsSL https://sourceforge.net/projects/bowtie-bio/files/bowtie2/${bowtie2_version}/bowtie2-${bowtie2_version}-source.zip -o bowtie2-${bowtie2_version}-source.zip \ +&& unzip bowtie2-${bowtie2_version}-source.zip \ +&& make -C bowtie2-${bowtie2_version} prefix=/usr/local install \ +&& rm -r bowtie2-${bowtie2_version} \ +&& rm bowtie2-${bowtie2_version}-source.zip + +RUN curl -fsSL https://github.com/OpenGene/fastp/archive/v${fastp_version}.tar.gz | tar -xz \ +&& cd fastp-${fastp_version} \ +&& make \ +&& make install \ +&& cd .. \ +&& rm -r fastp-${fastp_version} + +RUN wget https://www.bioinformatics.babraham.ac.uk/projects/fastqc/fastqc_v${fastqc_version}.zip \ +&& unzip fastqc_v${fastqc_version}.zip \ +&& chmod +x FastQC/fastqc \ +&& mv FastQC/* /usr/local/bin \ +&& rm fastqc_v${fastqc_version}.zip \ +&& rm -r FastQC + +RUN curl -fsSL https://github.com/DerrickWood/kraken2/archive/v${kraken2_version}.tar.gz | tar -xz \ +&& cd kraken2-${kraken2_version} \ +&& ./install_kraken2.sh /usr/local/bin \ +&& cd .. + +RUN curl -fsSL https://github.com/ArthurVM/Afanc/archive/refs/tags/v${afanc_version}-alpha.tar.gz | tar -xz \ +&& cd Afanc-${afanc_version}-alpha \ +&& pip3 install ./ \ +&& cd .. \ +&& curl -fsSL "https://github.com/marbl/Mash/releases/download/v${mash_version}/mash-Linux64-v${mash_version}.tar" | tar -x \ +&& mv mash-Linux64-v${mash_version}/mash /usr/local/bin \ +&& rm -r mash-Linux* \ +&& wget https://github.com/ParBLiSS/FastANI/releases/download/v${fastani_version}/fastANI-Linux64-v${fastani_version}.zip \ +&& unzip fastANI-Linux64-v${fastani_version}.zip \ +&& mv fastANI /usr/local/bin + +RUN sh -c "$(curl -fsSL ftp://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/install-edirect.sh)" \ +&& mkdir -p /opt/edirect \ +&& mv /root/edirect/* /opt/edirect + +RUN git clone --recursive -b geno_kmer_count https://github.com/phelimb/mccortex \ +&& make -C mccortex \ +&& mv mccortex /usr/local/bin \ +&& curl -fsSL mykrobe-${mykrobe_version}.tar.gz https://github.com/Mykrobe-tools/mykrobe/archive/v${mykrobe_version}.tar.gz | tar -xz \ +&& cd mykrobe-${mykrobe_version} \ +&& pip3 install requests \ +&& pip3 install . \ +&& ln -s /usr/local/bin/mccortex/bin/mccortex31 /usr/local/lib/python3.8/dist-packages/mykrobe/cortex/mccortex31 \ +&& mykrobe panels update_metadata \ +&& mykrobe panels update_species all \ +&& cd .. + +RUN curl -fsSL https://github.com/lh3/bwa/archive/v${bwa_version}.tar.gz | tar -C /usr/local/bin -xz \ +&& make -C /usr/local/bin/bwa-${bwa_version} \ +&& chmod +x /usr/local/bin/bwa-${bwa_version}/bwa + +RUN unset DEBIAN_FRONTEND + +ENV LC_ALL en_US.UTF-8 \ +LANG en_US.UTF-8 \ +LANGUAGE en_US.UTF-8 + diff --git a/test_scripts/mainscripts/docker/Dockerfile.tbprofiler-0.9.8 b/test_scripts/mainscripts/docker/Dockerfile.tbprofiler-0.9.8 new file mode 100644 index 0000000..686c9c4 --- /dev/null +++ b/test_scripts/mainscripts/docker/Dockerfile.tbprofiler-0.9.8 @@ -0,0 +1,54 @@ +FROM mambaorg/micromamba:1.3.0 as app + +#copy the reference genome to pre-compute our index +COPY resources/tuberculosis.fasta /data/tuberculosis.fasta + +USER root +WORKDIR / + +ARG TBPROFILER_VER="5.0.1" + +# this version is the shortened commit hash on the `master` branch here https://github.com/jodyphelan/tbdb/ +# commits are found on https://github.com/jodyphelan/tbdb/commits/master +# this was the latest commit as of 2023-10-26 +ARG TBDB_VER="e25540b" + +# LABEL instructions tag the image with metadata that might be important to the user +LABEL base.image="micromamba:1.3.0" +LABEL dockerfile.version="1" +LABEL software="tbprofiler" +LABEL software.version="${TBPROFILER_VER}" +LABEL description="The pipeline aligns reads to the H37Rv reference using bowtie2, BWA or minimap2 and then calls variants using bcftools. These variants are then compared to a drug-resistance database." +LABEL website="https://github.com/jodyphelan/TBProfiler/" +LABEL license="https://github.com/jodyphelan/TBProfiler/blob/master/LICENSE" +LABEL maintainer="John Arnn" +LABEL maintainer.email="jarnn@utah.gov" +LABEL maintainer2="Curtis Kapsak" +LABEL maintainer2.email="kapsakcj@gmail.com" + +# Install dependencies via apt-get; cleanup apt garbage +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget \ + ca-certificates \ + procps && \ + apt-get autoclean && rm -rf /var/lib/apt/lists/* + +# install tb-profiler via bioconda; install into 'base' conda env +RUN micromamba install --yes --name base --channel conda-forge --channel bioconda \ + tb-profiler=${TBPROFILER_VER} + +RUN micromamba install --yes --name base --channel conda-forge --channel bioconda gatk4 +RUN micromamba install --yes --name base --channel conda-forge --channel bioconda samtools +RUN micromamba install --yes --name base --channel conda-forge jq +RUN micromamba clean --all --yes + +# hardcode 'base' env bin into PATH, so conda env does not have to be "activated" at run time +ENV PATH="/opt/conda/bin:${PATH}" + +# Version of database can be confirmed at /opt/conda/share/tbprofiler/tbdb.version.json +# can also run 'tb-profiler list_db' to find the same version info +# In 5.0.1 updating_tbdb does not work with tb-profiler update_tbdb --commit ${TBDB_VER} +RUN tb-profiler update_tbdb --commit ${TBDB_VER} + +WORKDIR /data +RUN tb-profiler update_tbdb --match_ref tuberculosis.fasta diff --git a/test_scripts/mainscripts/docker/Dockerfile.vcfpredict-0.9.8 b/test_scripts/mainscripts/docker/Dockerfile.vcfpredict-0.9.8 new file mode 100644 index 0000000..9de1bb7 --- /dev/null +++ b/test_scripts/mainscripts/docker/Dockerfile.vcfpredict-0.9.8 @@ -0,0 +1,27 @@ +FROM ubuntu:20.04 + +LABEL maintainer="pricea35@cardiff.ac.uk" \ +about.summary="container for the vcf predict workflow" + +#add run-vcf to container +COPY bin/ /opt/bin/ +ENV PATH=/opt/bin:$PATH + +ENV PACKAGES="procps curl wget git build-essential libhdf5-dev libffi-dev r-base-core jq" \ +PYTHON="python3 python3-pip python3-dev" + +ENV vcfmix_version=d4693344bf612780723e39ce27c8ae3868f95417 + +#apt updates +RUN apt-get update \ +&& DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata \ +&& apt-get install -y $PACKAGES $PYTHON \ +&& apt-get install -y python3-packaging \ +&& git clone https://github.com/JeremyWesthead/VCFMIX.git \ +&& cd VCFMIX \ +&& git checkout ${vcfmix_version} \ +&& pip3 install recursive_diff \ +&& pip3 install awscli \ +&& pip3 install . \ +&& cp -r data /usr/local/lib/python3.8/dist-packages \ +&& cd .. diff --git a/test_scripts/mainscripts/nextflow.config b/test_scripts/mainscripts/nextflow.config new file mode 100644 index 0000000..43a0d71 --- /dev/null +++ b/test_scripts/mainscripts/nextflow.config @@ -0,0 +1,124 @@ +params { + + // help message + help = '' + + // input directory + input_dir = "" + + // filetype fastq or bam + filetype = 'fastq' + + // glob pattern for input files + pattern = '*_R{1,2}.fastq.gz' + + // output directory + output_dir = "" + + // unmix myco 'yes' or 'no' + unmix_myco = 'no' + + // principal species in each sample, assuming genus Mycobacterium, default null + // otherwise takes 1 of 10 values: abscessus, africanum, avium, bovis, chelonae, chimaera, fortuitum, intracellulare, kansasii, tuberculosis + species = 'null' + + // path to kraken2 database + kraken_db = "kraken2" + + // path to bowtie2 index + bowtie2_index = "bowtie2" + + // name of the bowtie index, e.g. hg19_1kgmaj + bowtie_index_name = "hg19_1kgmaj" + + // run VCFMIX 'yes' or 'no' (set to no for synthetic samples) + vcfmix = 'yes' + + resistance_profiler = "tb-profiler" + update_tbprofiler = "true" + + // path to singularity recipes directory (needed to strip software versions in getversion) + sing_dir = "${baseDir}/singularity" + + // path to this config file (needed to strip database versions in getversion) + config_dir = "${baseDir}/nextflow.config" + + // path to the Afanc Myco database + afanc_myco_db = "${baseDir}/Mycobacteriaciae_DB_7.0/" + + //path to resources directory + resource_dir = "${baseDir}/resources" + refseq = "${resource_dir}/assembly_summary_refseq.txt" + container_enabled = "false" +} + +profiles { + climb { + includeConfig 'config/containers.config' + + //add in docker configs as the above config file is generic for any containerised run + docker.enabled = true + fixOwnership = true + runOptions = "-u \$(id -u):\$(id -g)" + + withLabel:getversion{ + executor = "local" + container = null + } + + withLabel:afanc_parse{ + executor = "local" + container = null + } + + k8s { + computeResourceType = 'Job' + } + + //params specific to paths on the climb system + params{ + bowtie2_index = "s3://microbial-bioin-sp3/bowtie_hg19" + bowtie_index_name = "hg19_1kgmaj" + kraken_db = "s3://microbial-bioin-sp3/kraken_pluspf_16gb/" + afanc_myco_db = "s3://microbial-bioin-sp3/Mycobacteriaciae_DB_7.0/" + + resource_dir = "s3://microbial-bioin-sp3/lodestone_resources" + refseq = "${resource_dir}/assembly_summary_refseq.txt" + } + } + + singularity { + includeConfig 'config/containers.config' + + singularity.enabled = 'true' + singularity.autoMounts = 'true' + + //path to the singularity containers + singularity.cacheDir = "${baseDir}/singularity" + } + + sp3 { + + includeConfig 'config/containers.config' + + //add in singularity configs as the above config file is generic for any containerised run + singularity.enabled = 'true' + singularity.autoMounts = 'true' + // path to the singularity containers + singularity.cacheDir = "/data/images" + + process { + scratch = true + } + } + + + docker { + includeConfig 'config/containers.config' + + //add in docker configs as the above config file is generic for any containerised run + docker.enabled = true + fixOwnership = true + runOptions = "-u \$(id -u):\$(id -g)" + } +} diff --git a/test_scripts/workflows/TM01.nf b/test_scripts/workflows/TM01.nf index 5130f3c..d1636d0 100644 --- a/test_scripts/workflows/TM01.nf +++ b/test_scripts/workflows/TM01.nf @@ -2,35 +2,29 @@ nextflow.enable.dsl = 2 // import modules -include {checkFqValidity} from '../../tb-pipeline/modules/preprocessingModules.nf' params(params) -include {countReads} from '../../tb-pipeline/modules/preprocessingModules.nf' params(params) -include {fastp} from '../../tb-pipeline/modules/preprocessingModules.nf' params(params) -include {fastQC} from '../../tb-pipeline/modules/preprocessingModules.nf' params(params) -include {kraken2} from '../../tb-pipeline/modules/preprocessingModules.nf' params(params) -include {mykrobe} from '../../tb-pipeline/modules/preprocessingModules.nf' params(params) -include {bowtie2} from '../../tb-pipeline/modules/preprocessingModules.nf' params(params) -include {identifyBacterialContaminants} from '../../tb-pipeline/modules/preprocessingModules.nf' params(params) -include {downloadContamGenomes} from '../../tb-pipeline/modules/preprocessingModules.nf' params(params) -include {mapToContamFa} from '../../tb-pipeline/modules/preprocessingModules.nf' params(params) -include {reKraken} from '../../tb-pipeline/modules/preprocessingModules.nf' params(params) -include {reMykrobe} from '../../tb-pipeline/modules/preprocessingModules.nf' params(params) -include {summarise} from '../../tb-pipeline/modules/preprocessingModules.nf' params(params) -include {checkBamValidity} from '../../tb-pipeline/modules/preprocessingModules.nf' params(params) -include {bam2fastq} from '../../tb-pipeline/modules/preprocessingModules.nf' params(params) +include {checkFqValidity} from '../../lodestone/modules/preprocessingModules.nf' params(params) +include {countReads} from '../../lodestone/modules/preprocessingModules.nf' params(params) +include {fastp} from '../../lodestone/modules/preprocessingModules.nf' params(params) +include {fastQC} from '../../lodestone/modules/preprocessingModules.nf' params(params) +include {checkBamValidity} from '../../lodestone/modules/preprocessingModules.nf' params(params) +include {bam2fastq} from '../../lodestone/modules/preprocessingModules.nf' params(params) +include {getversion} from '../../lodestone/modules/getversionModules.nf' params(params) + // define workflow component workflow tm01 { take: input_files - krakenDB - bowtie_dir - + main: + + getversion() + input_files_vjson = input_files.combine(getversion.out.getversion_json) if ( params.filetype == "bam" ) { - checkBamValidity(input_files) + checkBamValidity(input_files_vjson) bam2fastq(checkBamValidity.out.checkValidity_bam) @@ -39,7 +33,7 @@ workflow tm01 { if ( params.filetype == "fastq" ) { - checkFqValidity(input_files) + checkFqValidity(input_files_vjson) countReads(checkFqValidity.out.checkValidity_fqs) }