nextflow.config

/*
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    nf-core/meerpipe Nextflow config file
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    Default config options for all compute environments
----------------------------------------------------------------------------------------
*/

manifest {
    // The metadata of the pipeline
    name = 'MeerPipe'
    homePage = 'https://github.com/OZGrav/meerpipe'
    description = 'Pulsar timing data processing pipeline for MeerTime data.'
    mainScript = 'meerpipe.nf'
    version = '3.0.6'
    author = 'Nick Swainston, Andrew Cameron, Aditya Parthasarathy, Stefan Oslowski, Andrew Jameson, Renée Spiewak, Daniel Reardon, Matthew Bailes'
    defaultBranch = 'main'
    doi = '10.5281/zenodo.7918680'
}
params.manifest = manifest





// Global default params, used in configs
params {
    params.input = null // Not used, but required for nf-core/configs

    // Observation selection options
    obs_csv  = ""
    utcs     = ""
    utce     = ""
    project  = ""
    pulsar   = ""

    // Processing options
    use_prev_ar = false // If true grab a previous archive file from output directory
    use_edge_subints = false // Use first and last 8 second subints of observation archives
    use_max_nsub = true // Use the maximum number of subints possible for the observation based on the S/N ratio
    use_all_nsub = true // Use the all nsubs, do not time scrunch
    use_mode_nsub = true // Use most common observation duration of the pulsar to calculate nsub
    tos_sn = 12 // Desired TOA S/N ratio, used to calculate the nsub to use
    nchans = "1,8,16,32,928" // List of nchans to frequency scrunch the data into
    max_nchan_upload = 32 // Residuals with nchan above this value will not be uploaded
    npols = "1,4" // List of number of polarisations to scrunch the data into
    chop_edge = true // Chop edge frequency channels


    // MeerTime database options
    upload = true  // Upload result to the database
    psrdb_url   = "$PSRDB_URL" // URL for interacting with the database API
    psrdb_token = "$PSRDB_TOKEN" // Token taken from enviroment variable and obtained using get_ingest_token.sh or get_token.sh


    // Directory options (input/output)
    input_dir = "/fred/oz005/timing" // Path to the directories for each pulsar
    outdir    = "/fred/oz005/timing_processed" //Path where the data products are stored

    // Ephermiris and template locations. This can be overridden by a user to use none default files
    ephemeris = ""
    template  = ""

    // Boilerplate options
    tracedir                   = "${params.outdir}/pipeline_info"
    publish_dir_mode           = 'copy'
    email                      = null
    email_on_fail              = 'ecarli@swin.edu.au'
    plaintext_email            = false
    monochrome_logs            = false
    hook_url                   = null
    help                       = false
    version                    = false
    validate_params            = true
    show_hidden_params         = false
    schema_ignore_params       = 'genomes,validationSchemaIgnoreParams,manifest,validation-schema-ignore-params'
    validationShowHiddenParams = false
    validationSchemaIgnoreParams = false


    // Config options
    config_profile_name        = null
    config_profile_description = null
    custom_config_version      = 'master'
    custom_config_base         = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}"
    config_profile_contact     = null
    config_profile_url         = null

    // Max resource options
    // Defaults only, expecting to be overwritten
    max_memory                 = '250.GB'
    max_cpus                   = 16
    max_time                   = '240.h'

    // Schema validation default options
    validationFailUnrecognisedParams = false
    validationLenientMode            = false
    validationSchemaIgnoreParams     = 'genomes,igenomes_base'
    validationShowHiddenParams       = false
    validate_params                  = true

}

// Load base.config by default for all pipelines
includeConfig 'conf/base.config'

// Load nf-core custom profiles from different Institutions
try {
    includeConfig "${params.custom_config_base}/nfcore_custom.config"
} catch (Exception e) {
    System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config")
}

// Load nf-core/meerpipe custom profiles from different institutions.
// Warning: Uncomment only if a pipeline-specific institutional config already exists on nf-core/configs!
// try {
//   includeConfig "${params.custom_config_base}/pipeline/meerpipe.config"
// } catch (Exception e) {
//   System.err.println("WARNING: Could not load nf-core/config/meerpipe profiles: ${params.custom_config_base}/pipeline/meerpipe.config")
// }

def archive_size_calc(nchan, nbin, dur) {
    // Return the size of the archive in GB
    return 1e-09 * Float.valueOf(nchan) * Float.valueOf(nbin) * Float.valueOf(dur)
}

def mem_calc(factor, task_attempt, nchan, nbin, dur, ram_max) {
    // Return memory required by the job MB
    // factor: is the factor to multiply the archive size by to get the memory required
    // task_attempt: is the number of attempts the task has had
    // nchan: is the number of frequency channels
    // nbin: is the number of profile bins
    // dur: duration is in seconds
    // ram_max is maximum ram available for th job in GB

    archive_size_gb = archive_size_calc(nchan, nbin, dur)

    mem = factor ** task_attempt * archive_size_gb + 2

    ram_max = ram_max.split("\\.")[0].toFloat() - 1
    if ( mem > ram_max ) {
        mem = ram_max
    }
    return (int) ( mem )
}

// Software versions
psrdb_version = "3.0.6"
meerpipe_version = "71bd66c"
tempo2_version = "ce14d72_temponest_apar_56e2b58"
tempo2_clock_version = "2021-04-26"
psrchive_version = "4c0597b7b"
meertime_ephemerides_and_templates_version = "bd021f5"


// CLUSTER SPECFIC DEFAULTS
// ----------------------------------------------------------------------------
def hostname = "hostname".execute().text.trim().replace("-", "")
params.hostname = hostname
if ( hostname.startsWith("tooarrana") || hostname.startsWith("trevor") ) {
    // Set up for OzStars new Ngarrgu Tindebeek cluster

    // Set up containers
    // process.module = 'apptainer'
    // singularity {
    //     enabled = true
    //     runOptions = '--nv -B /nvmetmp'
    //     envWhitelist = 'SINGULARITY_BINDPATH, SINGULARITYENV_LD_LIBRARY_PATH'
    // }
    // params.containerDir = '/pawsey/mwa/singularity'

    // Max resources for the milan partition
    params {
        max_memory                 = '512.GB'
        max_cpus                   = 62
        max_time                   = '7.day'
    }

    // Default directories
    workDir = "/fred/oz005/users/${USER}/work"

    process {
        cache = 'lenient'

        if ( hostname.startsWith("tooarrana") ) {
            module_use_command = "module use /apps/users/pulsar/milan/gcc-11.3.0/modulefiles; module use /apps/users/pulsar/common/modulefiles;"
        }
        else if ( hostname.startsWith("trevor") ) {
            module_use_command = "export SYS_ARCH=openstack; module use /apps/users/pulsar/openstack/gcc-11.3.0/modulefiles; module use /apps/users/pulsar/common/modulefiles;"
        }

        // Resource set up
        withLabel: process_single {
            maxForks = 1
            errorStrategy = 'retry'
            maxRetries = 2
        }
        withLabel: database_upload {
            maxForks = 128
            errorStrategy = 'retry'
            maxRetries = 2
        }
        withLabel: process_low {
            executor = 'slurm'
            errorStrategy = 'retry'
            maxRetries = 2
            queue  = 'milan'
            cpus   = 1
            time   = "600 s"
            memory = "1 GB"
        }
        withLabel: process_medium {
            executor = 'slurm'
            errorStrategy = 'retry'
            maxRetries = 2
            queue  = 'milan'
            cpus   = 1
            time   = { "${task.attempt * 1.0 * meta.dur.toFloat() + 300} s" }
            memory = { "${mem_calc(3, task.attempt, meta.obs_nchan, meta.obs_nbin, meta.dur, params.max_memory)} GB" }
        }
        withLabel: process_high {
            executor = 'slurm'
            errorStrategy = 'retry'
            maxRetries = 2
            queue  = 'milan'
            cpus   = 1
            time   = { "${task.attempt * 1.5 * meta.dur.toFloat() * meta.obs_nchan.toFloat()/1024 + 300} s" }
            memory = { "${mem_calc(9, task.attempt, meta.obs_nchan, meta.obs_nbin, meta.dur, params.max_memory)} GB" }
        }
        withLabel: scratch {
            scratch = '$JOBFS'
            stageInMode = 'copy'
            clusterOptions = { "--tmp=${(task.attempt * archive_size_calc(meta.obs_nchan, meta.obs_nbin, meta.dur) * 6000).toInteger() + 50}MB" }
        }
        // Software dependency set up
        withLabel: psrdb {
            beforeScript = "${module_use_command} module load psrdb/${psrdb_version}"
        }
        withLabel: psrdb_tempo2 {
            beforeScript = "${module_use_command} module load psrdb/${psrdb_version} tempo2/${tempo2_version} tempo2_clock/${tempo2_clock_version}"
        }
        withLabel: meerpipe {
            beforeScript = "module use /apps/users/pulsar/milan/gcc-11.3.0/modulefiles; module use /apps/users/pulsar/common/modulefiles; module load meerpipe/${meerpipe_version}"
        }
        withLabel: psrchive {
            beforeScript = "module use /apps/users/pulsar/milan/gcc-11.3.0/modulefiles; module use /apps/users/pulsar/common/modulefiles; module load psrchive/${psrchive_version}"
        }
        withLabel: ephem_template {
            beforeScript = "${module_use_command} module load meertime_ephemerides_and_templates/${meertime_ephemerides_and_templates_version}"
        }
    }
    executor.submitRateLimit = '100 sec'
    executor.exitReadTimeout = '6 hours'
    executor.$slurm.queueSize = 1000

}
else if ( hostname.startsWith("farnarkle") ) {
    // Set up for Swinburnes's Ozstar cluster
}
else {
    // No recognised hostname so assuming defaults

    // Resource set up
    executor.name = 'local'
    executor.queueSize = 8

    // Set up containers
    docker.enabled = true
}


profiles {
    debug {
        dumpHashes             = true
        process.beforeScript   = 'echo $HOSTNAME'
        cleanup                = false
        nextflow.enable.configProcessNamesValidation = true
    }
    conda {
        conda.enabled          = true
        docker.enabled         = false
        singularity.enabled    = false
        podman.enabled         = false
        shifter.enabled        = false
        charliecloud.enabled   = false
        channels               = ['conda-forge', 'bioconda', 'defaults']
        apptainer.enabled      = false
    }
    mamba {
        conda.enabled          = true
        conda.useMamba         = true
        docker.enabled         = false
        singularity.enabled    = false
        podman.enabled         = false
        shifter.enabled        = false
        charliecloud.enabled   = false
        apptainer.enabled      = false
    }
    docker {
        docker.enabled         = true
        conda.enabled          = false
        singularity.enabled    = false
        podman.enabled         = false
        shifter.enabled        = false
        charliecloud.enabled   = false
        apptainer.enabled      = false
        docker.runOptions      = "-u \$(id -u):\$(id -g) -v ${params.input_dir}:${params.input_dir}"
    }
    arm {
        docker.runOptions      = '-u $(id -u):$(id -g) --platform=linux/amd64'
    }
    singularity {
        singularity.enabled    = true
        singularity.autoMounts = true
        conda.enabled          = false
        docker.enabled         = false
        podman.enabled         = false
        shifter.enabled        = false
        charliecloud.enabled   = false
        apptainer.enabled      = false
    }
    podman {
        podman.enabled         = true
        conda.enabled          = false
        docker.enabled         = false
        singularity.enabled    = false
        shifter.enabled        = false
        charliecloud.enabled   = false
        apptainer.enabled      = false
    }
    shifter {
        shifter.enabled        = true
        conda.enabled          = false
        docker.enabled         = false
        singularity.enabled    = false
        podman.enabled         = false
        charliecloud.enabled   = false
        apptainer.enabled      = false
    }
    charliecloud {
        charliecloud.enabled   = true
        conda.enabled          = false
        docker.enabled         = false
        singularity.enabled    = false
        podman.enabled         = false
        shifter.enabled        = false
        apptainer.enabled      = false
    }
    apptainer {
        apptainer.enabled      = true
        apptainer.autoMounts   = true
        conda.enabled          = false
        docker.enabled         = false
        singularity.enabled    = false
        podman.enabled         = false
        shifter.enabled        = false
        charliecloud.enabled   = false
    }
    gitpod {
        executor.name          = 'local'
        executor.cpus          = 4
        executor.memory        = 8.GB
    }
    test      { includeConfig 'conf/test.config'      }
    test_full { includeConfig 'conf/test_full.config' }
    devel {
        params {
            psrdb_url   = "http://localhost:8000/"
            psrdb_token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VybmFtZSI6Im5pY2siLCJleHAiOjE3MDQ4NzU3MjYsIm9yaWdJYXQiOjE3MDQ4NzU0MjZ9.Wg0jYvHRHVuGv9xbQeFhvZXjfP3EdrWTRIanUaPTd4k"
        }
        process {
            // Software dependency set up
            withLabel: psrdb {
                beforeScript = "module use /apps/users/pulsar/milan/gcc-11.3.0/modulefiles; module use /apps/users/pulsar/common/modulefiles; module load meertime_ephemerides_and_templates/${meertime_ephemerides_and_templates_version}; source ~/venv/bin/activate"
            }
            withLabel: psrdb_tempo2 {
                beforeScript = "module use /apps/users/pulsar/milan/gcc-11.3.0/modulefiles; module use /apps/users/pulsar/common/modulefiles; module load tempo2/${tempo2_version}; source ~/venv/bin/activate"
            }
            withLabel: meerpipe {
                beforeScript = "module use /apps/users/pulsar/milan/gcc-11.3.0/modulefiles; module use /apps/users/pulsar/common/modulefiles; module load scintools/1334918 meertime_ephemerides_and_templates/${meertime_ephemerides_and_templates_version}; source ~/venv/bin/activate"
            }
            withLabel: psrchive {
                beforeScript = "module use /apps/users/pulsar/milan/gcc-11.3.0/modulefiles; module use /apps/users/pulsar/common/modulefiles; module load psrchive/${psrchive_version}"
            }
            withLabel: ephem_template {
                beforeScript = "module use /apps/users/pulsar/milan/gcc-11.3.0/modulefiles; module load meertime_ephemerides_and_templates/${meertime_ephemerides_and_templates_version}"
            }
        }
    }
}

// Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile
// Will not be used unless Apptainer / Docker / Podman / Singularity are enabled
// Set to your registry if you have a mirror of containers
apptainer.registry   = 'quay.io'
docker.registry      = 'docker.io'
podman.registry      = 'quay.io'
singularity.registry = 'quay.io'

// Nextflow plugins
plugins {
    id 'nf-validation@1.1.3' // Validation of pipeline parameters and creation of an input channel from a sample sheet
}

// Export these variables to prevent local Python/R libraries from conflicting with those in the container
// The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container.
// See https://apeltzer.github.io/post/03-julia-lang-nextflow/ for details on that. Once we have a common agreement on where to keep Julia packages, this is adjustable.

env {
    PYTHONNOUSERSITE = 1
    R_PROFILE_USER   = "/.Rprofile"
    R_ENVIRON_USER   = "/.Renviron"
    JULIA_DEPOT_PATH = "/usr/local/share/julia"
}

// Capture exit codes from upstream processes when piping
process.shell = ['/bin/bash', '-euo', 'pipefail']

// Disable process selector warnings by default. Use debug profile to enable warnings.
nextflow.enable.configProcessNamesValidation = false

def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')
timeline {
    enabled = true
    file    = "${params.outdir}/pipeline_info/execution_timeline_${trace_timestamp}.html"
}
report {
    enabled = true
    file    = "${params.outdir}/pipeline_info/execution_report_${trace_timestamp}.html"
}
trace {
    enabled = true
    file    = "${params.outdir}/pipeline_info/execution_trace_${trace_timestamp}.txt"
    overwrite = true
}
dag {
    enabled = true
    file    = "${params.outdir}/pipeline_info/pipeline_dag_${trace_timestamp}.html"
}

manifest {
    name            = 'nf-core/meerpipe'
    author          = """Nick Swainston"""
    homePage        = 'https://github.com/nf-core/meerpipe'
    description     = """Pulsar timing data processing pipeline for MeerTime data."""
    mainScript      = 'main.nf'
    nextflowVersion = '!>=23.04.0'
    version         = '1.0dev'
    doi             = ''
}

// Load modules.config for DSL2 module specific options
includeConfig 'conf/modules.config'

// Function to ensure that resource requirements don't go beyond
// a maximum limit
def check_max(obj, type) {
    if (type == 'memory') {
        try {
            if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1)
                return params.max_memory as nextflow.util.MemoryUnit
            else
                return obj
        } catch (all) {
            println "   ### ERROR ###   Max memory '${params.max_memory}' is not valid! Using default value: $obj"
            return obj
        }
    } else if (type == 'time') {
        try {
            if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1)
                return params.max_time as nextflow.util.Duration
            else
                return obj
        } catch (all) {
            println "   ### ERROR ###   Max time '${params.max_time}' is not valid! Using default value: $obj"
            return obj
        }
    } else if (type == 'cpus') {
        try {
            return Math.min( obj, params.max_cpus as int )
        } catch (all) {
            println "   ### ERROR ###   Max cpus '${params.max_cpus}' is not valid! Using default value: $obj"
            return obj
        }
    }
}