cleanup

LICENSE.txt

@@ -0,0 +1,26 @@
+# ===========================================================================
+#
+#                            PUBLIC DOMAIN NOTICE
+#                     Center for Information Technology (CIT)
+#                        National Institute of Health (NIH)
+#
+#  This software/database is a "United States Government Work" under the
+#  terms of the United States Copyright Act.  It was written as part of
+#  the author's official duties as a United States Government employee and
+#  thus cannot be copyrighted.  This software is freely available
+#  to the public for use.  The Center for Information Technology, The
+#  National Institutes of Health, and the U.S. Government have not placed
+#  any restriction on its use or reproduction.
+#
+#  Although all reasonable efforts have been taken to ensure the accuracy
+#  and reliability of the software and data, CIT, NIH and the U.S.
+#  Government do not and cannot warrant the performance or results that
+#  may be obtained by using this software or data. CIT, NIH and the U.S.
+#  Government disclaim all warranties, express or implied, including
+#  warranties of performance, merchantability or fitness for any particular
+#  purpose.
+#
+#  Please cite the author and the "NIH Biowulf Cluster" in any work or product
+#  based on this material.
+#
+# ===========================================================================

README.md

@@ -1,12 +1,65 @@
-pbs2slurm notes
-================================================================================
+Translating batch scripts with `#PBS` directives into Slurm scripts with `#SBATCH` directives
+=============================================================================================
+`pbs2slurm.py` translates PBS type batch scripts to Slurm. It translates directives
+and a limited subset of environment variables.
+
+`pbs2slurm_tests.py` serves two purposes: It takes a number of test cases and
+checks for correct translation and outputs the results as an html table in
+`testcases.html`.
+
+### Usage
+
+```
+usage: pbs2slurm [-h] [--shell SHELL] [--version] [pbs_script]
+
+Translates PBS batch script to Slurm.
+
+The PBS script is split into
+- a shebang line
+- a header containing #PBS directives, comments, and empty lines
+- the body of the script
+
+pbs2slurm carries out 3 transformation steps
+- if no shebang line was present in the PBS script, a new one is added. By 
+  default this is #! /bin/bash, but this can be changed (see below)
+- #PBS directives in the header are translated, where possible, to #SBATCH 
+  directives.
+- common PBS environment variables in the body are translated to their SLURM 
+  equivalents
+
+Please be sure to manually go over translated scripts to esure their 
+correctness.
+
+If no input file is specified, pbs2slurm reads from stdin. The translated script 
+is written to stdout.
+
+Examples:
+    pbs2slurm < pbs_script > slurm_script
+    pbs2slurm pbs_script > slurm_script
+    pbs2slurm -s /bin/zsh pbs_script > slurm_script
+
+See also https://hpc.cit.nih.gov/docs/pbs2slurm.html.
+
+positional arguments:
+  pbs_script
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --shell SHELL, -s SHELL
+                        Shell to insert if shebang line (#! ...) is missing.
+                        Defaults to '/bin/bash'
+  --version, -v
+```
+
+### pbs2slurm notes
 
 - PBS directives in batch script use a more relaxed
   grammar than command line switches. For example
-      '#PBS -N foo'
-      '#PBS -Nfoo'
-      '#PBS-Nfoo'
-      all work! All of these will be correctly translated.
+    -  `#PBS -N foo`
+    -  `#PBS -Nfoo`
+    -  `#PBS-Nfoo`
+  all work! All of these will be correctly translated.
+
 - to get an idea of what is in PBS job scripts i collected
   40119 pbs directives from existing scripts. Frequencies
   of the different directives:
@@ -34,11 +87,13 @@ pbs2slurm notes
     457 #PBS -k oe
     ```
   where -k oe results in out and err files in the user's home
+
 - this script is not computationally efficient with all the
   repeated string substitutions. However, efficiency doesn't 
   really matter for this script.
+
 - this script does not gracefully deal with multiple occurences
   of the same options. It generally translates them all. This 
   could lead to some confusion for scripts that use, for example,
-  #PBS -V and #PBS -v, but those are rare (and that's kind of iffy
+  `#PBS -V` and `#PBS -v`, but those are rare (and that's kind of iffy
   to begin with).

examples/ex1.sh

@@ -3,11 +3,10 @@
 #PBS -N
 #PBS -k n
 #PBS -m n
-#PBS -M somebody@helix.nih.gov
+#PBS -M somebody@somewhere.net
 #PBS -j oe
 #PBS -o /path/to/output
 #PBS -V
-# parameter sweep for swembl using wt CTCFab
 
 
 function sw()  {

pbs2slurm_tests.py

@@ -3,10 +3,6 @@
 import atexit
 import difflib
 
-sys.stderr = sys.stdout
-
-html = open("testcases.html", "w")
-
 def html_out(fh, pbs, slurm, desc):
     pbss = pbs.replace(">", "&gt;").replace("<", "&lt;")
     slurms = slurm.replace(">", "&gt;").replace("<", "&lt;")
@@ -23,14 +19,28 @@ def html_out(fh, pbs, slurm, desc):
 
 def html_header(fh):
     fh.write("""
-<table id="test_output">
-    <tr><th><b>PBS script</b></th>
-        <th><b>SLURM script</b></th>
-    </tr>
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+      <meta charset="utf-8">
+      <title>pbs2slurm test cases</title>
+      <style>pre.term {background-color: #eee; padding: 3px 5px; border: 1px solid #999}</style>
+  </head>
+  <body>
+    <!-- START page content -->
+    <table id="test_output">
+        <tr><th><b>PBS script</b></th>
+            <th><b>SLURM script</b></th>
+        </tr>
 """)
 
-html_header(html)
-
+def html_footer(fh):
+    fh.write("""
+    </table>
+    <!-- END page content -->
+  </body>
+</html>
+""")
 
 def check(input, exp, obs, desc):
     if exp != obs:
@@ -983,3 +993,57 @@ def test_script4():
     """
     check(input, expected, p2s.convert_batch_script(input), desc)
 
+if __name__ == '__main__':
+    # this is a pretty stupid way of doing this - should have used a testing
+    # framework
+    testfunctions = (
+        test_plain_bash,
+        test_pbs_o_workdir,
+        test_pbs_jobid,
+        test_pbs_arrayid,
+        test_missing_shebang,
+        test_header_identification,
+        test_jobname,
+        test_jobname_empty,
+        test_valid_email_address,
+        test_empty_email_address,
+        test_multiple_email_addresses,
+        test_multiple_email_addresses2,
+        test_email_modes_n,
+        test_email_modes_a,
+        test_email_modes_b,
+        test_email_modes_e,
+        test_email_modes_be,
+        test_email_modes_abe,
+        test_email_modes_aben,
+        test_email_modes_empty,
+        test_keep_directive_ignored,
+        test_join_directive_ignored_eo,
+        test_stdout_directive,
+        test_empty_stdout_directive,
+        test_stderr_directive,
+        test_empty_stderr_directive,
+        test_restartable_directive_y,
+        test_restartable_directive_n,
+        test_restartable_directive_bad,
+        test_drop_shell_directive,
+        test_export_whole_env,
+        test_export_individual_variables_1,
+        test_export_individual_variables_2,
+        test_export_individual_variables_2,
+        test_fix_job_array,
+        test_drop_empty_job_array,
+        test_resources,
+        test_drop_queue,
+        test_script1,
+        test_script2,
+        test_script3,
+        test_script4,
+    )
+    sys.stderr = sys.stdout
+    html = open("testcases.html", "w")
+    html_header(html)
+    for testf in testfunctions:
+        testf()
+    html_footer(html)
+    html.close()