From bcfcd718c537038a95a282b1d1f4a591ba75495e Mon Sep 17 00:00:00 2001 From: Travis Horine Date: Thu, 3 Jun 2021 11:23:14 -0500 Subject: [PATCH 1/3] Update __init__.py The addition of an 'immediate' function call feature to the JobCluster's setup logic allows subsequent items in the 'depends' list to include code that cannot otherwise be defined due to missing module imports at the global level... i.e. declaring a custom class that itself extends a builtin class which must be imported as a base class. --- py3/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/py3/__init__.py b/py3/__init__.py index 4e1c01f..e3cb261 100644 --- a/py3/__init__.py +++ b/py3/__init__.py @@ -2660,7 +2660,7 @@ def __init__(self, computation, nodes=None, depends=[], job_status=None, cluster if inspect.isfunction(setup): if setup.__defaults__: print('\n dispy does not support calling "setup" with keyword arguments\n') - depends.append(setup) + depends.insert(0, setup) compute.setup = setup.__name__ compute.setup_args_count = setup.__code__.co_argcount elif isinstance(setup, functools.partial): @@ -2761,8 +2761,9 @@ def __init__(self, computation, nodes=None, depends=[], job_status=None, cluster elif (inspect.isfunction(dep) or inspect.isclass(dep) or (hasattr(dep, '__class__') and hasattr(dep, '__module__'))): + immediate=None if inspect.isfunction(dep) or inspect.isclass(dep): - pass + immediate = dep.__name__ if dep.__name__.startswith('nodeinit_') else None elif hasattr(dep, '__class__') and inspect.isclass(dep.__class__): dep = dep.__class__ try: @@ -2772,6 +2773,8 @@ def __init__(self, computation, nodes=None, depends=[], job_status=None, cluster raise lines[0] = lines[0].lstrip() compute.code += '\n' + ''.join(lines) + if immediate: + compute.code += '\n' + immediate + '()\n' elif isinstance(dep, functools.partial): try: lines = inspect.getsourcelines(dep.func)[0] From 511ac126806062711793a4a4ceb9ac9d8ece0417 Mon Sep 17 00:00:00 2001 From: Travis Horine Date: Thu, 3 Jun 2021 11:27:48 -0500 Subject: [PATCH 2/3] Update obj_instances.py Modified example to show the potential need for a 'nodeinit_setup' which executes immediately after it is serialized to the node. This feature means that classes like the class 'C' in the example can extend other python builtin classes like the 'cmd' class. --- py3/examples/obj_instances.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/py3/examples/obj_instances.py b/py3/examples/obj_instances.py index 0ac7c21..e6238af 100644 --- a/py3/examples/obj_instances.py +++ b/py3/examples/obj_instances.py @@ -1,6 +1,7 @@ # example program that sends object instances in local program # as arguments to distributed computation -class C: +import cmd +class C(cmd.Cmd): def __init__(self, i, n): self.i = i self.n = n @@ -17,7 +18,11 @@ def compute(obj): if __name__ == '__main__': import random, dispy - cluster = dispy.JobCluster(compute, depends=[C]) + def nodeinit_setup(): + global cmd + import cmd + return 0 + cluster = dispy.JobCluster(compute, depends=[C], setup=nodeinit_setup) jobs = [] for i in range(10): c = C(i, random.uniform(1, 3)) # create object of C From 09aff8315adafa523144e856fd16915616b4e1e6 Mon Sep 17 00:00:00 2001 From: Travis Horine Date: Thu, 3 Jun 2021 11:37:11 -0500 Subject: [PATCH 3/3] Update obj_instances.py --- py3/examples/obj_instances.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/py3/examples/obj_instances.py b/py3/examples/obj_instances.py index e6238af..9696ca9 100644 --- a/py3/examples/obj_instances.py +++ b/py3/examples/obj_instances.py @@ -1,6 +1,8 @@ # example program that sends object instances in local program # as arguments to distributed computation +# Also shows use of nodeinit_ feature to ensure that cmd module is immediately imported so that class C can extend cmd.Cmd on the node import cmd + class C(cmd.Cmd): def __init__(self, i, n): self.i = i