|
29 | 29 | from .context import RuntimeContext, getdefault
|
30 | 30 | from .cuda import cuda_version_and_device_count
|
31 | 31 | from .cwlprov.provenance_profile import ProvenanceProfile
|
32 |
| -from .errors import WorkflowException |
| 32 | +from .errors import WorkflowException, WorkflowKillSwitch |
33 | 33 | from .job import JobBase
|
34 | 34 | from .loghandler import _logger
|
35 | 35 | from .mutation import MutationManager
|
@@ -260,6 +260,11 @@ def run_jobs(
|
260 | 260 | WorkflowException,
|
261 | 261 | ): # pylint: disable=try-except-raise
|
262 | 262 | raise
|
| 263 | + except WorkflowKillSwitch as err: |
| 264 | + _logger.error( |
| 265 | + f"Workflow kill switch activated by [job {err.job_id}] " |
| 266 | + f"because on-error={runtime_context.on_error}" |
| 267 | + ) |
263 | 268 | except Exception as err:
|
264 | 269 | logger.exception("Got workflow error")
|
265 | 270 | raise WorkflowException(str(err)) from err
|
@@ -332,6 +337,11 @@ def _runner(
|
332 | 337 | except WorkflowException as err:
|
333 | 338 | _logger.exception(f"Got workflow error: {err}")
|
334 | 339 | self.exceptions.append(err)
|
| 340 | + except WorkflowKillSwitch as err: |
| 341 | + _logger.error( |
| 342 | + f"Workflow kill switch activated by [job {err.job_id}] " |
| 343 | + f"because on-error={runtime_context.on_error}" |
| 344 | + ) |
335 | 345 | except Exception as err: # pylint: disable=broad-except
|
336 | 346 | _logger.exception(f"Got workflow error: {err}")
|
337 | 347 | self.exceptions.append(WorkflowException(str(err)))
|
@@ -466,9 +476,8 @@ def run_jobs(
|
466 | 476 | while self.taskqueue.in_flight > 0:
|
467 | 477 | self.wait_for_next_completion(runtime_context)
|
468 | 478 | self.run_job(None, runtime_context)
|
469 |
| - |
470 |
| - runtime_context.workflow_eval_lock.release() |
471 | 479 | finally:
|
| 480 | + runtime_context.workflow_eval_lock.release() |
472 | 481 | self.taskqueue.drain()
|
473 | 482 | self.taskqueue.join()
|
474 | 483 |
|
|
0 commit comments