reworked config modules activation in an execution context

and ease use of external software by re-introducing in_context (#338, #335)

Author: denisri

capsul/config/afni.py

@@ -1,5 +1,5 @@
 from .configuration import ModuleConfiguration
-from soma.controller import Directory, undefined, File, field
+from soma.controller import Directory, undefined, field
 
 
 class AfniConfiguration(ModuleConfiguration):
@@ -15,11 +15,11 @@ def is_valid_config(self, requirements):
             return False
         return True
 
+    @staticmethod
+    def init_execution_context(execution_context):
+        """
+        Configure execution (env variables) from a configured execution context
+        """
+        from capsul.in_context import afni
 
-def init_execution_context(execution_context):
-    """
-    Configure an execution context given a capsul_engine and some requirements.
-    """
-    config = execution_context.config["modules"]["afni"]
-    execution_context.afni = AfniConfiguration()
-    execution_context.afni.import_from_dict(config)
+        afni.set_env_from_config(execution_context)

capsul/config/ants.py

@@ -1,5 +1,5 @@
 from .configuration import ModuleConfiguration
-from soma.controller import Directory, undefined, File, field
+from soma.controller import Directory, undefined, field
 
 
 class AntsConfiguration(ModuleConfiguration):
@@ -15,11 +15,11 @@ def is_valid_config(self, requirements):
             return False
         return True
 
+    @staticmethod
+    def init_execution_context(execution_context):
+        """
+        Configure execution (env variables) from a configured execution context
+        """
+        from capsul.in_context import ants
 
-def init_execution_context(execution_context):
-    """
-    Configure an execution context given a capsul_engine and some requirements.
-    """
-    config = execution_context.config["modules"]["ants"]
-    execution_context.ants = AntsConfiguration()
-    execution_context.ants.import_from_dict(config)
+        ants.set_env_from_config(execution_context)

capsul/config/axon.py

@@ -50,12 +50,3 @@ def axon_default_shared_dir():
     @staticmethod
     def axon_default_version():
         return axon_default_version()
-
-
-def init_execution_context(execution_context):
-    """
-    Configure an execution context given a capsul_engine and some requirements.
-    """
-    config = execution_context.config["modules"]["axon"]
-    execution_context.axon = AxonConfiguration()
-    execution_context.axon.import_from_dict(config)

capsul/config/configuration.py

@@ -191,6 +191,9 @@ def add_module(self, module_name, allow_existing=False):
             doc=cls.__doc__,
             default_factory=OpenKeyDictController[cls],
         )
+        if self.config_modules is undefined:
+            self.config_modules = []
+        self.config_modules.append(module_name)
 
         if hasattr(cls, "module_dependencies"):
             module_dependencies = getattr(cls, "module_dependencies")

capsul/config/freesurfer.py

@@ -16,11 +16,11 @@ def is_valid_config(self, requirements):
             return False
         return True
 
+    @staticmethod
+    def init_execution_context(execution_context):
+        """
+        Configure execution (env variables) from a configured execution context
+        """
+        from capsul.in_context import freesurfer
 
-def init_execution_context(execution_context):
-    """
-    Configure an execution context given a capsul_engine and some requirements.
-    """
-    config = execution_context.config["modules"]["freesurfer"]
-    execution_context.freesurfer = FreesurferConfiguration()
-    execution_context.freesurfer.import_from_dict(config)
+        freesurfer.set_env_from_config(execution_context)

capsul/config/fsl.py

@@ -17,11 +17,11 @@ def is_valid_config(self, requirements):
             return False
         return True
 
+    @staticmethod
+    def init_execution_context(execution_context):
+        """
+        Configure execution (env variables) from a configured execution context
+        """
+        from capsul.in_context import fsl
 
-def init_execution_context(execution_context):
-    """
-    Configure an execution context given a capsul_engine and some requirements.
-    """
-    config = execution_context.config["modules"]["fsl"]
-    execution_context.fsl = FSLConfiguration()
-    execution_context.fsl.import_from_dict(config)
+        fsl.set_env_from_config(execution_context)

capsul/config/matlab.py

@@ -50,11 +50,11 @@ def is_valid_config(self, requirements):
                 return False
         return True
 
-
-def init_execution_context(execution_context):
-    """
-    Configure an execution context given a capsul_engine and some requirements.
-    """
-    config = execution_context.config["modules"]["matlab"]
-    execution_context.matlab = MatlabConfiguration()
-    execution_context.matlab.import_dict(config)
+    @staticmethod
+    def init_execution_context(execution_context):
+        """
+        Configure execution (env variables) from a configured execution context
+        """
+        from capsul.in_context import matlab
+
+        matlab.set_env_from_config(execution_context)

capsul/config/mrtrix.py

@@ -0,0 +1,25 @@
+from .configuration import ModuleConfiguration
+from soma.controller import Directory, undefined, field
+
+
+class MRTrixConfiguration(ModuleConfiguration):
+    """MRTrix configuration module"""
+
+    version: str
+    directory: Directory = field(optional=True)
+    name = "mrtrix"
+
+    def is_valid_config(self, requirements):
+        required_version = requirements.get("version")
+        if required_version and getattr(self, "version", undefined) != required_version:
+            return False
+        return True
+
+    @staticmethod
+    def init_execution_context(execution_context):
+        """
+        Configure execution (env variables) from a configured execution context
+        """
+        from capsul.in_context import mrtrix
+
+        mrtrix.set_env_from_config(execution_context)

capsul/config/nipype.py

@@ -154,7 +154,7 @@ def configure_freesurfer(context):
                 freesurfer.FSCommand.set_default_subjects_dir(subjects_dir)
             from capsul.in_context import freesurfer as fsrun
 
-            env = fsrun.freesurfer_env()
+            env = fsrun.freesurfer_env(execution_context=context)
             for var, value in env.items():
                 os.environ[var] = value
 

capsul/config/python.py

@@ -16,12 +16,3 @@ def is_valid_config(self, requirements):
         if required_version and getattr(self, "version", undefined) != required_version:
             return False
         return True
-
-
-def init_execution_context(execution_context):
-    """
-    Configure an execution context given a capsul_engine and some requirements.
-    """
-    config = execution_context.config["modules"]["python"]
-    execution_context.python = PythonConfiguration()
-    execution_context.python.import_from_dict(config)

capsul/config/spm.py

@@ -21,11 +21,11 @@ def is_valid_config(self, requirements):
         else:
             return {"matlab": {"mcr": False}}
 
+    @staticmethod
+    def init_execution_context(execution_context):
+        """
+        Configure execution (env variables) from a configured execution context
+        """
+        from capsul.in_context import spm
 
-def init_execution_context(execution_context):
-    """
-    Configure an execution context given a capsul_engine and some requirements.
-    """
-    config = execution_context.config["modules"]["spm"]
-    execution_context.spm = SPMConfiguration()
-    execution_context.spm.import_dict(config)
+        spm.set_env_from_config(execution_context)

capsul/engine/__init__.py

@@ -27,7 +27,9 @@ def execution_context(engine_label, engine_config, executable):
     for conf_item in ("dataset", "config_modules", "python_modules"):
         if conf_item in cdict:
             config[conf_item] = cdict[conf_item]
-    execution_context = ExecutionContext(executable=executable, config=config)
+    execution_context = ExecutionContext(
+        executable=executable, config=config, activate_modules=False
+    )
 
     req_to_check = execution_context.executable_requirements(executable)
     done_req = []  # record requirements to avoid loops
@@ -60,7 +62,7 @@ def execution_context(engine_label, engine_config, executable):
     # now check we have only one module for each
     for module_name in needed_modules:
         valid_module_configs = valid_configs.get(module_name)
-        if not valid_module_configs:
+        if valid_module_configs is None:
             raise RuntimeError(
                 f'Execution environment "{engine_label}" has no '
                 f"valid configuration for module {module_name}"
@@ -81,6 +83,10 @@ def execution_context(engine_label, engine_config, executable):
             module_name, type_=ModuleConfiguration, override=True
         )
         setattr(execution_context, module_name, valid_config)
+
+    # FIXME: should be done only in real execution (server) situation
+    execution_context.activate_modules_config()
+
     return execution_context
 
 

capsul/execution_context.py

@@ -25,17 +25,17 @@ class ExecutionContext(Controller):
     config_modules: list[str]
     dataset: OpenKeyDictController[Dataset]
 
-    def __init__(self, config=None, executable=None):
-        mod_classes = []
+    def __init__(self, config=None, executable=None, activate_modules=True):
+        super().__init__()
         if config:
             python_modules = config.get("python_modules", ())
             for m in python_modules:
                 importlib.import_module(m)
             config_modules = config.get("config_modules", ())
+            self.config_modules = config_modules
             for m in config_modules:
                 # The following function loads the appropriate module
                 get_config_class(m)
-        super().__init__()
         self.dataset = OpenKeyDictController[Dataset]()
         if config is not None:
             for k in list(config.keys()):
@@ -53,14 +53,17 @@ def __init__(self, config=None, executable=None):
                     k = new_k
                 if cls:
                     self.add_field(k, cls, doc=cls.__doc__, default_factory=cls)
-                    mod_classes.append(cls)
             dataset = config.pop("dataset", None)
             if dataset:
                 self.dataset = dataset
             self.import_dict(config)
         self.executable = executable
+        if activate_modules:
+            self.activate_modules_config()
 
-        for cls in mod_classes:
+    def activate_modules_config(self):
+        for cm in self.config_modules:
+            cls = get_config_class(cm)
             if hasattr(cls, "init_execution_context"):
                 cls.init_execution_context(self)
 

capsul/in_context/__init__.py

@@ -1,15 +1,19 @@
 # -*- coding: utf-8 -*-
 """
-The ``in_context`` module provides functions to call some external software from Capsul processes (SPM, FSL, etc.). The main functions perform calls to the software in a similar way as ``subprocess`` functions (:class:`~subprocess.Popen`, :func:`~subprocess.call`, :func:`~subprocess.check_call` and :func:`subprocess.check_output`). These functions are only valid when the software environment *context* is activated.
-Activating the context is normally done using the ``with`` statement on a :class:`~capsul.engine.CapsulEngine` object::
+The ``in_context`` module provides functions to call some external software from Capsul processes (SPM, FSL, etc.). The main functions perform calls to the software in a similar way as ``subprocess`` functions (:class:`~subprocess.Popen`, :func:`~subprocess.call`, :func:`~subprocess.check_call` and :func:`subprocess.check_output`).
+The notable difference is that they use an :class:`~capsul.execution_context.ExecutionContext` object instance to get configuration from.
+These functions are only run from within the :meth:`~capsul.process.Process.execute` method of a Process, which gets the context as a paremeter::
 
-    from capsul.engine import capsul_engine
+    from capsul.api import Process
     from capsul.in_context.fsl import fsl_check_call
 
-    ce = capsul_engine()
+    ce = Capsul()
     # .. configure it ...
+    Class MyProcess(Process):
 
-    with ce:
-        fsl_check_call(['bet', '-h'])
+        # [declare fields etc] ...
+
+        def execute(self, execution_context):
+            fsl_check_call(['bet', '-h'], execution_context=execution_context)
 
 """