diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..75d7e1a --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,26 @@ +# =========================================================================== +# +# PUBLIC DOMAIN NOTICE +# Center for Information Technology (CIT) +# National Institute of Health (NIH) +# +# This software/database is a "United States Government Work" under the +# terms of the United States Copyright Act. It was written as part of +# the author's official duties as a United States Government employee and +# thus cannot be copyrighted. This software is freely available +# to the public for use. The Center for Information Technology, The +# National Institutes of Health, and the U.S. Government have not placed +# any restriction on its use or reproduction. +# +# Although all reasonable efforts have been taken to ensure the accuracy +# and reliability of the software and data, CIT, NIH and the U.S. +# Government do not and cannot warrant the performance or results that +# may be obtained by using this software or data. CIT, NIH and the U.S. +# Government disclaim all warranties, express or implied, including +# warranties of performance, merchantability or fitness for any particular +# purpose. +# +# Please cite the author and the "NIH Biowulf Cluster" in any work or product +# based on this material. +# +# =========================================================================== diff --git a/README.md b/README.md index 152ec83..219cb5a 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,65 @@ -pbs2slurm notes -================================================================================ +Translating batch scripts with `#PBS` directives into Slurm scripts with `#SBATCH` directives +============================================================================================= +`pbs2slurm.py` translates PBS type batch scripts to Slurm. It translates directives +and a limited subset of environment variables. + +`pbs2slurm_tests.py` serves two purposes: It takes a number of test cases and +checks for correct translation and outputs the results as an html table in +`testcases.html`. + +### Usage + +``` +usage: pbs2slurm [-h] [--shell SHELL] [--version] [pbs_script] + +Translates PBS batch script to Slurm. + +The PBS script is split into +- a shebang line +- a header containing #PBS directives, comments, and empty lines +- the body of the script + +pbs2slurm carries out 3 transformation steps +- if no shebang line was present in the PBS script, a new one is added. By + default this is #! /bin/bash, but this can be changed (see below) +- #PBS directives in the header are translated, where possible, to #SBATCH + directives. +- common PBS environment variables in the body are translated to their SLURM + equivalents + +Please be sure to manually go over translated scripts to esure their +correctness. + +If no input file is specified, pbs2slurm reads from stdin. The translated script +is written to stdout. + +Examples: + pbs2slurm < pbs_script > slurm_script + pbs2slurm pbs_script > slurm_script + pbs2slurm -s /bin/zsh pbs_script > slurm_script + +See also https://hpc.cit.nih.gov/docs/pbs2slurm.html. + +positional arguments: + pbs_script + +optional arguments: + -h, --help show this help message and exit + --shell SHELL, -s SHELL + Shell to insert if shebang line (#! ...) is missing. + Defaults to '/bin/bash' + --version, -v +``` + +### pbs2slurm notes - PBS directives in batch script use a more relaxed grammar than command line switches. For example - '#PBS -N foo' - '#PBS -Nfoo' - '#PBS-Nfoo' - all work! All of these will be correctly translated. + - `#PBS -N foo` + - `#PBS -Nfoo` + - `#PBS-Nfoo` + all work! All of these will be correctly translated. + - to get an idea of what is in PBS job scripts i collected 40119 pbs directives from existing scripts. Frequencies of the different directives: @@ -34,11 +87,13 @@ pbs2slurm notes 457 #PBS -k oe ``` where -k oe results in out and err files in the user's home + - this script is not computationally efficient with all the repeated string substitutions. However, efficiency doesn't really matter for this script. + - this script does not gracefully deal with multiple occurences of the same options. It generally translates them all. This could lead to some confusion for scripts that use, for example, - #PBS -V and #PBS -v, but those are rare (and that's kind of iffy + `#PBS -V` and `#PBS -v`, but those are rare (and that's kind of iffy to begin with). diff --git a/examples/ex1.sh b/examples/ex1.sh index 96e0712..43d2b1b 100644 --- a/examples/ex1.sh +++ b/examples/ex1.sh @@ -3,11 +3,10 @@ #PBS -N #PBS -k n #PBS -m n -#PBS -M somebody@helix.nih.gov +#PBS -M somebody@somewhere.net #PBS -j oe #PBS -o /path/to/output #PBS -V -# parameter sweep for swembl using wt CTCFab function sw() { diff --git a/pbs2slurm_tests.py b/pbs2slurm_tests.py index 020a7d1..ad0ea7f 100644 --- a/pbs2slurm_tests.py +++ b/pbs2slurm_tests.py @@ -3,10 +3,6 @@ import atexit import difflib -sys.stderr = sys.stdout - -html = open("testcases.html", "w") - def html_out(fh, pbs, slurm, desc): pbss = pbs.replace(">", ">").replace("<", "<") slurms = slurm.replace(">", ">").replace("<", "<") @@ -23,14 +19,28 @@ def html_out(fh, pbs, slurm, desc): def html_header(fh): fh.write(""" - - - - + + + + + pbs2slurm test cases + + + + +
PBS scriptSLURM script
+ + + """) -html_header(html) - +def html_footer(fh): + fh.write(""" +
PBS scriptSLURM script
+ + + +""") def check(input, exp, obs, desc): if exp != obs: @@ -983,3 +993,57 @@ def test_script4(): """ check(input, expected, p2s.convert_batch_script(input), desc) +if __name__ == '__main__': + # this is a pretty stupid way of doing this - should have used a testing + # framework + testfunctions = ( + test_plain_bash, + test_pbs_o_workdir, + test_pbs_jobid, + test_pbs_arrayid, + test_missing_shebang, + test_header_identification, + test_jobname, + test_jobname_empty, + test_valid_email_address, + test_empty_email_address, + test_multiple_email_addresses, + test_multiple_email_addresses2, + test_email_modes_n, + test_email_modes_a, + test_email_modes_b, + test_email_modes_e, + test_email_modes_be, + test_email_modes_abe, + test_email_modes_aben, + test_email_modes_empty, + test_keep_directive_ignored, + test_join_directive_ignored_eo, + test_stdout_directive, + test_empty_stdout_directive, + test_stderr_directive, + test_empty_stderr_directive, + test_restartable_directive_y, + test_restartable_directive_n, + test_restartable_directive_bad, + test_drop_shell_directive, + test_export_whole_env, + test_export_individual_variables_1, + test_export_individual_variables_2, + test_export_individual_variables_2, + test_fix_job_array, + test_drop_empty_job_array, + test_resources, + test_drop_queue, + test_script1, + test_script2, + test_script3, + test_script4, + ) + sys.stderr = sys.stdout + html = open("testcases.html", "w") + html_header(html) + for testf in testfunctions: + testf() + html_footer(html) + html.close()