Jebuss/walltime

.gitignore

@@ -1,8 +1,59 @@
-*.pyc
-*.pyo
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
 
-*.egg
-gridmap.egg-info/
-dist/
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
 build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
 docs/_build/
+
+# PyBuilder
+target/

README.md

@@ -0,0 +1,33 @@
+#GridMap
+
+This is a WIP fork of the original gridmap project. This was created to achieve compatability
+with the TORQUE and PBS grid engine.
+
+### Requirements
+Jobs can only be submitted from nodes than are allowed to do that (i.e they can run 'qsub')
+
+A couple of environment variables need to be set in order to work.
+
+ERROR_MAIL_RECIPIENT = *your email address*
+
+export DRMAA_LIBRARY_PATH = *like pbs_drmaa/libs/libdrmaa.so for pbs*
+
+export DEFAULT_TEMP_DIR="/local/$USER/"
+
+export USE_MEM_FREE=TRUE
+
+export SMTP_SERVER="unimail.tu-dortmund.de"
+
+export ERROR_MAIL_RECIPIENT="[email protected]"
+
+export ERROR_MAIL_SENDER="[email protected]"
+
+export SEND_ERROR_MAIL=TRUE
+
+### Python Requirements
+
+
+-  drmaa <https://github.com/drmaa-python/drmaa-python>
+-  psutil <https://github.com/giampaolo/psutil>
+-  pyzmq <https://github.com/zeromq/pyzmq>
+-  Python 3.4+

examples/manual.py

@@ -32,8 +32,15 @@
 from datetime import datetime
 
 from gridmap import Job, process_jobs
-
-
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--engine', help='Name of the grid engine you are using.', choices=['TOURQUE','PBS','SGE'], default='SGE')
+parser.add_argument('--queue', help='Name of the queue you want to send jobs to.', default='all.q')
+parser.add_argument('--vmem', help='Amount of memory to use on a node.', default='200m')
+parser.add_argument('--port', help='The port through which to communicate with the JobMonitor', default=None, type=int)
+parser.add_argument('--local', help='Flag indicating whether the jobs should run locally instead of on the cluster', default=False, type=bool)
+parser.add_argument("--logging", type=str, choices=['INFO', 'DEBUG', 'WARN'], help='increase output verbosity', default='INFO')
 def sleep_walk(secs):
     '''
     Pass the time by adding numbers until the specified number of seconds has
@@ -46,18 +53,18 @@ def sleep_walk(secs):
         num = num + 1
 
 
-def compute_factorial(n):
+def compute_factorial(n, sleep=10):
     """
     computes factorial of n
     """
-    sleep_walk(10)
+    sleep_walk(sleep)
     ret = 1
     for i in range(n):
         ret = ret * (i + 1)
     return ret
 
 
-def make_jobs():
+def make_jobs(engine, queue, vmem):
     """
     creates a list of Job objects,
     which carry all information needed
@@ -68,7 +75,7 @@ def make_jobs():
     """
 
     # set up list of arguments
-    inputvec = [[3], [5], [10], [20]]
+    inputvec = [[3, 10], [5, 20], [10, 10], [20, 20]]
 
     # create empty job vector
     jobs = []
@@ -77,7 +84,8 @@ def make_jobs():
     for arg in inputvec:
         # The default queue used by the Job class is all.q. You must specify
         # the `queue` keyword argument if that is not the name of your queue.
-        job = Job(compute_factorial, arg, queue='all.q')
+        job = Job(compute_factorial, arg, queue=queue, engine=engine,
+                                    mem_free=vmem)
         jobs.append(job)
 
     return jobs
@@ -88,21 +96,37 @@ def main():
     run a set of jobs on cluster
     """
 
+    args = parser.parse_args()
+    engine = args.engine
+    queue = args.queue
+    vmem = args.vmem
+    port = args.port
+    local =args.local
+    level = args.logging
+
+    if level is 'DEBUG':
+        level = logging.DEBUG
+    elif level is 'WARN':
+        level = logging.WARN
+    elif level is 'INFO':
+        level = logging.INFO
+
     logging.captureWarnings(True)
-    logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
-                                '%(message)s'), level=logging.INFO)
+    logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +  '%(message)s'), level=level)
 
     print("=====================================")
     print("========   Submit and Wait   ========")
-    print("=====================================")
-    print("")
+    print("=====================================\n")
+
 
-    functionJobs = make_jobs()
+    functionJobs = make_jobs(engine, queue, vmem)
+    if local :
+        print('Running jobs locally')
+    else:
+        print("Sending function jobs to cluster engine: {}. Into queue: {} \n".format(engine, queue))
 
-    print("sending function jobs to cluster")
-    print("")
 
-    job_outputs = process_jobs(functionJobs, max_processes=4)
+    job_outputs = process_jobs(functionJobs, max_processes=4, port=port, local=local)
 
     print("results from each job")
     for (i, result) in enumerate(job_outputs):
@@ -111,4 +135,3 @@ def main():
 
 if __name__ == "__main__":
     main()
-

gridmap/conf.py

@@ -75,10 +75,11 @@
 try:
     import drmaa
     DRMAA_PRESENT = True
-except (ImportError, OSError, RuntimeError):
+except (ImportError, OSError, RuntimeError) as e:
     logger = logging.getLogger(__name__)
     logger.warning('Could not import drmaa. Only local multiprocessing ' +
                    'supported.')
+    logger.warning(str(e))
     DRMAA_PRESENT = False
 
 # plot cpu and mem usage and send via email