Implement job cpu and memory resource requests and limits for jobs

Author: manning-ncsa

envvars.py

@@ -46,5 +46,9 @@
 DR2_TILE_HOST_PATH = os.environ['DR2_TILE_HOST_PATH']
 JOB_UID = os.environ.get('JOB_UID', '1001')
 JOB_GID = os.environ.get('JOB_GID', '1001')
+JOB_RESOURCE_REQUEST_CPU    = os.environ.get('JOB_RESOURCE_REQUEST_CPU',    '100m')
+JOB_RESOURCE_LIMIT_CPU      = os.environ.get('JOB_RESOURCE_LIMIT_CPU',      '1')
+JOB_RESOURCE_REQUEST_MEMORY = os.environ.get('JOB_RESOURCE_REQUEST_MEMORY', '100Mi')
+JOB_RESOURCE_LIMIT_MEMORY   = os.environ.get('JOB_RESOURCE_LIMIT_MEMORY',   '1Gi')
 
 FRONTEND_BASE_PATH = re.sub(r'http.*\/\/{}'.format(BASE_DOMAIN), '', FRONTEND_BASE_URL)

job.tpl.yaml

@@ -89,6 +89,8 @@ spec:
               command: ["python3", "prestop.py"]
         resources:
           limits:
-            cpu: {{ resource_limit_cpu}}
+            cpu: {{ resource_limit_cpu | quote }}
+            memory: {{ resource_limit_memory | quote }}
           requests:
-            cpu: {{ resource_request_cpu}}
+            cpu: {{ resource_request_cpu | quote }}
+            memory: {{ resource_request_memory | quote }}

jobutils.py

@@ -1959,8 +1959,10 @@ def submit_job(params):
     conf['configjob']['spec'] = {}
 
     # TODO: Variate the resource requests/limits between the task types
-    conf["resource_limit_cpu"] = 1
-    conf["resource_request_cpu"] = 1
+    conf["resource_limit_cpu"] = envvars.JOB_RESOURCE_LIMIT_CPU
+    conf["resource_request_cpu"] = envvars.JOB_RESOURCE_REQUEST_CPU
+    conf["resource_limit_memory"] = envvars.JOB_RESOURCE_LIMIT_MEMORY
+    conf["resource_request_memory"] = envvars.JOB_RESOURCE_REQUEST_MEMORY
 
 
     # Should the job be executed synchronously or in an independent container (Kubernetes Job)

kubejob.py

@@ -57,6 +57,8 @@ def job(input):
         username=input["configjob"]["metadata"]["username"],
         resource_limit_cpu=input["resource_limit_cpu"],
         resource_request_cpu=input["resource_request_cpu"],
+        resource_limit_memory=input["resource_limit_memory"],
+        resource_request_memory=input["resource_request_memory"],
         desarchiveHostPath=envvars.DESARCHIVE_HOST_PATH,
         coaddHostPath=envvars.COADD_HOST_PATH,
         dr2TilePath=envvars.DR2_TILE_HOST_PATH,