diff --git a/capsul/attributes/completion_engine_iteration.py b/capsul/attributes/completion_engine_iteration.py index fb235dcb3..d40d977ae 100644 --- a/capsul/attributes/completion_engine_iteration.py +++ b/capsul/attributes/completion_engine_iteration.py @@ -254,6 +254,8 @@ def complete_parameters(self, process_inputs={}, print('assign iteration parameter', parameter, ':\n', e, file=sys.stderr) for parameter in parameters: + if parameter in iterative_parameters: + continue try: value = getattr(process.process, parameter) setattr(process, parameter, value) diff --git a/capsul/engine/__init__.py b/capsul/engine/__init__.py index de6eb4d2c..e4629d6d7 100644 --- a/capsul/engine/__init__.py +++ b/capsul/engine/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" This module defines the main API to interact with Capsul processes. In order to execute a process, it is mandatory to have an instance of :py:class:`CapsulEngine`. Such an instance can be created with factory @@ -18,18 +18,13 @@ --------------------- :func:`activate_configuration` ------------------------------ -''' +""" -from __future__ import absolute_import -from __future__ import print_function import importlib -import json import os import os.path as osp import re import tempfile -import subprocess -import sys from traits.api import Dict, String, Undefined @@ -38,7 +33,6 @@ from soma.sorted_dictionary import SortedDictionary from soma.utils.weak_proxy import get_ref -from .database_json import JSONDBEngine from .database_populse import PopulseDBEngine from .settings import Settings @@ -48,24 +42,24 @@ # FIXME TODO: OBSOLETE -#Questions about API/implementation: +# Questions about API/implementation: -#* execution: - #* workflows are not exposed, they are running a possibly different pipeline (single process case), thus we need to keep track on it - #* logging / history / provenance, databasing - #* retrieving output files with transfers: when ? currently in wait(), should it be a separate method ? should it be asynchronous ? - #* setting output parameters: currently in wait(), should it be a separate method ? - #* disconnections / reconnections client / server - #* actually connect computing resource[s] -#* settings / config: - #* see comments in settings.py - #* GUI and constraints on parameters ? - #* how to handle optional dependencies: ie nipype depends on spm if spm is installed / configured, otherwise we can run other nipype interfaces, but no spm ones - #* integrate soma-workflow config + CE.computing_resource +# * execution: +# * workflows are not exposed, they are running a possibly different pipeline (single process case), thus we need to keep track on it +# * logging / history / provenance, databasing +# * retrieving output files with transfers: when ? currently in wait(), should it be a separate method ? should it be asynchronous ? +# * setting output parameters: currently in wait(), should it be a separate method ? +# * disconnections / reconnections client / server +# * actually connect computing resource[s] +# * settings / config: +# * see comments in settings.py +# * GUI and constraints on parameters ? +# * how to handle optional dependencies: ie nipype depends on spm if spm is installed / configured, otherwise we can run other nipype interfaces, but no spm ones +# * integrate soma-workflow config + CE.computing_resource class CapsulEngine(Controller): - ''' + """ A CapsulEngine is the mandatory entry point of all software using Capsul. It contains objects to store configuration and metadata, defines execution environment(s) (possibly remote) and performs pipelines execution. @@ -183,40 +177,37 @@ class CapsulEngine(Controller): :class:`~capsul.engine.module.spm` **Methods** - ''' + """ - def __init__(self, - database_location, - database, - require): - ''' + def __init__(self, database_location, database, require): + """ CapsulEngine.__init__(self, database_location, database, config=None) The CapsulEngine constructor should not be called directly. Use :func:`capsul_engine` factory function instead. - ''' + """ super(CapsulEngine, self).__init__() - + self._settings = None - + self._database_location = database_location - self._database = database + self._database = database self._loaded_modules = set() self.load_modules(require) - + from capsul.study_config.study_config import StudyConfig + self.study_config = StudyConfig(engine=self) - self._metadata_engine = from_json(database.json_value('metadata_engine')) + self._metadata_engine = from_json(database.json_value("metadata_engine")) - self._connected_resource = '' - + self._connected_resource = "" @property def settings(self): if self._settings is None: - self._settings = Settings(self.database.db) + self._settings = Settings(self.database.storage) return self._settings @property @@ -226,49 +217,47 @@ def database(self): @property def database_location(self): return self._database_location - - + @property def metadata_engine(self): return self._metadata_engine - + @metadata_engine.setter def metadata_engine(self, metadata_engine): self._metadata_engine = metadata_engine - self.database.set_json_value('metadata_engine', - to_json(self._metadata_engine)) - + self.database.set_json_value("metadata_engine", to_json(self._metadata_engine)) + def load_modules(self, require): - ''' + """ Call self.load_module for each required module. The list of modules to load is located in self.modules (if it is None, capsul.module.default_modules is used). - ''' + """ if require is None: require = default_modules - + for module in require: self.load_module(module) def load_module(self, module_name): - ''' + """ Load a module if it has not already been loaded (is this case, nothing is done) - + A module is a fully qualified name of a Python module (as accepted by Python import statement). Such a module must define the two following functions (and may define two others, see below): - - def load_module(capsul_engine, module_name): + + def load_module(capsul_engine, module_name): def set_environ(config, environ): - + load_module of each module is called once before reading and applying the configuration. It can be used to add traits to the CapsulEngine in order to define the configuration options that are used by the module. Values of these traits are automatically stored in configuration in database when self.save() is used, and they are retrieved from database before initializing modules. - + set_environ is called in the context of the processing (i.e. on the, possibly remote, machine that runs the pipelines). It receives the configuration as a JSON compatible dictionary (for instance a @@ -277,13 +266,13 @@ def set_environ(config, environ): dictionary to set the environment variables that must be defined for pipeline configuration. These variables are typically used by modules in capsul.in_context module to manage running external - software with appropriate configuration. - ''' + software with appropriate configuration. + """ module_name = self.settings.module_name(module_name) if module_name not in self._loaded_modules: self._loaded_modules.add(module_name) python_module = importlib.import_module(module_name) - init_settings = getattr(python_module, 'init_settings', None) + init_settings = getattr(python_module, "init_settings", None) if init_settings is not None: init_settings(self) return True @@ -317,14 +306,14 @@ def path_metadata(self, path, named_directory=None): return self.database.set_path_metadata(path, named_directory) def import_configs(self, environment, config_dict, cont_on_error=False): - ''' + """ Import config values from a dictionary as given by :meth:`Settings.select_configurations`. Compared to :meth:`Settings.import_configs` this method (at :class:`CapsulEngine` level) also loads the required modules. - ''' - modules = config_dict.get('capsul_engine', {}).get('uses', {}) + """ + modules = config_dict.get("capsul_engine", {}).get("uses", {}) for module in modules: self.load_module(module) self.settings.import_configs(environment, config_dict, cont_on_error) @@ -333,19 +322,25 @@ def import_configs(self, environment, config_dict, cont_on_error=False): # Processes and pipelines related methods # def get_process_instance(self, process_or_id, **kwargs): - ''' + """ The only official way to get a process instance is to use this method. For now, it simply calls self.study_config.get_process_instance but it will change in the future. - ''' - instance = self.study_config.get_process_instance(process_or_id, - **kwargs) + """ + instance = self.study_config.get_process_instance(process_or_id, **kwargs) return instance - def get_iteration_pipeline(self, pipeline_name, node_name, process_or_id, - iterative_plugs=None, do_not_export=None, - make_optional=None, **kwargs): - """ Create a pipeline with an iteration node iterating the given + def get_iteration_pipeline( + self, + pipeline_name, + node_name, + process_or_id, + iterative_plugs=None, + do_not_export=None, + make_optional=None, + **kwargs + ): + """Create a pipeline with an iteration node iterating the given process. Parameters @@ -372,14 +367,14 @@ def get_iteration_pipeline(self, pipeline_name, node_name, process_or_id, pipeline = Pipeline() pipeline.name = pipeline_name pipeline.set_study_config(get_ref(self.study_config)) - pipeline.add_iterative_process(node_name, process_or_id, - iterative_plugs, do_not_export, - **kwargs) + pipeline.add_iterative_process( + node_name, process_or_id, iterative_plugs, do_not_export, **kwargs + ) pipeline.autoexport_nodes_parameters(include_optional=True) return pipeline def start(self, process, workflow=None, history=True, get_pipeline=False, **kwargs): - ''' + """ Asynchronously start the execution of a process or pipeline in the connected computing environment. Returns an identifier of the process execution and can be used to get the status of the @@ -412,74 +407,73 @@ def start(self, process, workflow=None, history=True, get_pipeline=False, **kwar execution identifier (actually a soma-workflow id) pipeline: Pipeline instance (optional) only returned if get_pipeline is True. - ''' + """ return run.start(self, process, workflow, history, get_pipeline, **kwargs) def connect(self, computing_resource): - ''' + """ Connect the capsul engine to a computing resource - ''' + """ self._connected_resource = computing_resource - def connected_to(self): - ''' + """ Return the name of the computing resource this capsul engine is connected to or None if it is not connected. - ''' + """ return self._connected_resource def disconnect(self): - ''' + """ Disconnect from a computing resource. - ''' + """ self._connected_resource = None def executions(self): - ''' + """ List the execution identifiers of all processes that have been started but not disposed in the connected computing resource. Raises an exception if the computing resource is not connected. - ''' + """ raise NotImplementedError() def dispose(self, execution_id, conditional=False): - ''' + """ Update the database with the current state of a process execution and - free the resources used in the computing resource (i.e. remove the + free the resources used in the computing resource (i.e. remove the workflow from SomaWorkflow). If ``conditional`` is set to True, then dispose is only done if the configuration does not specify to keep succeeded / failed workflows. - ''' + """ run.dispose(self, execution_id, conditional=conditional) def interrupt(self, execution_id): - ''' + """ Try to stop the execution of a process. Does not wait for the process to be terminated. - ''' + """ return run.interrupt(self, execution_id) def wait(self, execution_id, timeout=-1, pipeline=None): - ''' + """ Wait for the end of a process execution (either normal termination, interruption or error). - ''' + """ return run.wait(self, execution_id, timeout=timeout, pipeline=pipeline) def status(self, execution_id): - ''' - Return a simple value with the status of an execution (queued, + """ + Return a simple value with the status of an execution (queued, running, terminated, error, etc.) - ''' + """ return run.status(self, execution_id) def detailed_information(self, execution_id): - ''' + """ Return complete (and possibly big) information about a process execution. - ''' + """ return run.detailed_information(self, execution_id) def call(self, process, history=True, *kwargs): @@ -489,27 +483,40 @@ def check_call(self, process, history=True, **kwargs): return run.check_call(self, process, history=history, **kwargs) def raise_for_status(self, status, execution_id=None): - ''' + """ Raise an exception if a process execution failed - ''' + """ run.raise_for_status(self, status, execution_id) -_populsedb_url_re = re.compile(r'^\w+(\+\w+)?://(.*)') +_populsedb_url_re = re.compile(r"^\w+(\+\w+)?://(.*)") + + +class AutoDeleteFileName(str): + def __new__(cls, **kwargs): + fd, path = tempfile.mkstemp(**kwargs) + instance = super().__new__(cls, path) + instance.path = path + instance.fd = fd + return instance + + def __del__(self): + os.close(self.fd) + os.remove(self.path) def database_factory(database_location): - ''' + """ Create a DatabaseEngine from its location string. This location can be - either a sqlite file path (ending with '.sqlite' or ':memory:' for an + either a sqlite file path (ending with '.sqlite' or ':memory:' for an in memory database for testing) or a populse_db URL, or None. - ''' - global _populsedb_url_re + """ + global _populsedb_url_re engine_directory = None if database_location is None: - database_location = ':memory:' + database_location = AutoDeleteFileName(prefix="populse_db_", suffix=".sqlite") match = _populsedb_url_re.match(database_location) if match: path = match.groups(2) @@ -517,22 +524,23 @@ def database_factory(database_location): if path.startswith(os.apth.sep): engine_directory = osp.abspath(osp.dirname(path)) populse_db = database_location - elif database_location.endswith('.sqlite'): - populse_db = 'sqlite:///%s' % database_location + elif database_location.endswith(".sqlite"): + populse_db = f"sqlite://{database_location}" engine_directory = osp.abspath(osp.dirname(database_location)) - elif database_location == ':memory:': - populse_db = 'sqlite:///:memory:' + elif database_location == ":memory:": + populse_db = "sqlite://:memory:" else: - raise ValueError('Invalid database location: %s' % database_location) + raise ValueError("Invalid database location: %s" % database_location) engine = PopulseDBEngine(populse_db) if engine_directory: - engine.set_named_directory('capsul_engine', engine_directory) + engine.set_named_directory("capsul_engine", engine_directory) return engine + def capsul_engine(database_location=None, require=None): - ''' - User facrory for creating capsul engines. + """ + User factory for creating capsul engines. If no database_location is given, it will default to an internal (in- memory) database with no persistent settings or history values. @@ -542,16 +550,16 @@ def capsul_engine(database_location=None, require=None): database.json_value('global_config')), it contains the configuration values that are shared by all processing engines. The second entry is computing_config`. It contains a dictionary with one item per computing - resource where the key is the resource name and the value is configuration + resource where the key is the resource name and the value is configuration values that are specific to this computing resource. Before initialization of the CapsulEngine, modules are loaded. The list of loaded modules is searched in the 'modules' value in the database (i.e. in database.json_value('modules')) ; if no list is defined in the database, capsul.module.default_modules is used. - ''' - #if database_location is None: - #database_location = osp.expanduser('~/.config/capsul/capsul_engine.sqlite') + """ + # if database_location is None: + # database_location = osp.expanduser('~/.config/capsul/capsul_engine.sqlite') database = database_factory(database_location) capsul_engine = CapsulEngine(database_location, database, require=require) return capsul_engine @@ -560,30 +568,32 @@ def capsul_engine(database_location=None, require=None): configurations = None activated_modules = set() + def activate_configuration(selected_configurations): - ''' + """ Activate a selected configuration (set of modules) for runtime. - ''' + """ global configurations configurations = selected_configurations - modules = configurations.get('capsul_engine', {}).get('uses', {}).keys() + modules = configurations.get("capsul_engine", {}).get("uses", {}).keys() for m in modules: activate_module(m) + def activate_module(module_name): - ''' + """ Activate a module configuration for runtime. This function is called by activate_configuration() and assumes the global variable ``capsul.engine.configurations`` is properly setup. - ''' + """ global activated_modules if module_name not in activated_modules: activated_modules.add(module_name) module = importlib.import_module(module_name) - check_configurations = getattr(module, 'check_configurations', None) - complete_configurations = getattr(module, 'complete_configurations', None) + check_configurations = getattr(module, "check_configurations", None) + complete_configurations = getattr(module, "complete_configurations", None) if check_configurations: error = check_configurations() if error: @@ -592,6 +602,6 @@ def activate_module(module_name): error = check_configurations() if error: raise EnvironmentError(error) - activate_configurations = getattr(module, 'activate_configurations', None) + activate_configurations = getattr(module, "activate_configurations", None) if activate_configurations: activate_configurations() diff --git a/capsul/engine/database.py b/capsul/engine/database.py index c33e122b1..521b6ceae 100644 --- a/capsul/engine/database.py +++ b/capsul/engine/database.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import os.path as osp -class DatabaseEngine(object): +class DatabaseEngine: ''' A :py:class:`DatabaseEngine` is the base class for all engines allowing to store, retrieve and search metadata associated with diff --git a/capsul/engine/database_populse.py b/capsul/engine/database_populse.py index a52146569..2e4b855e4 100644 --- a/capsul/engine/database_populse.py +++ b/capsul/engine/database_populse.py @@ -1,159 +1,126 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import os.path as osp -import six -import uuid from capsul.engine.database import DatabaseEngine -from populse_db.database import Database +from populse_db.storage import Storage + +schemas = [ + { + "version": "1.0.0", + "schema": { + "named_directory": [ + { + "name": [str, {"primary_key": True}], + "path": str, + } + ], + "json_value": [ + { + "name": [str, {"primary_key": True}], + "json_dict": dict, + } + ], + "path_metadata": [ + { + "path": [str, {"primary_key": True}], + "named_directory": str, + } + ], + "metadata": [ + { + "path": [str, {"primary_key": True}], + "subject": str, + "time_point": str, + "history": list[str], # contains a list of execution_id + } + ], + }, + }, +] class PopulseDBEngine(DatabaseEngine): def __init__(self, database_engine): - self.db = Database(database_engine) - with self.db as dbs: - if not dbs.get_collection('path_metadata'): - # Create the schema if it does not exist - dbs.add_collection('named_directory', 'name') - dbs.add_field('named_directory', 'path', 'string') - - dbs.add_collection('json_value', 'name') - dbs.add_field('json_value', 'value', 'json') - - dbs.add_collection('path_metadata', 'path') - dbs.add_field('path_metadata', 'named_directory', 'string', - description='Reference to a base directory whose ' - 'path is stored in named_directory collection') - #self.dbs = self.db.__enter__() - - + self.storage = Storage(database_engine) + with self.storage.schema() as schema: + schema.add_schema("capsul.engine.database_populse") + def __del__(self): self.close() - - + def close(self): - #self.db.__exit__(None, None, None) - self.db = None - #self.dbs = None - - - def commit(self): - #self.dbs.save_modifications() - with self.db as dbs: - dbs.save_modifications() + self.storage = None - def rollback(self): - #self.dbs.unsave_modifications() - with self.db as dbs: - dbs.unsave_modifications() def set_named_directory(self, name, path): if path: path = osp.normpath(osp.abspath(path)) - #doc = self.dbs.get_document('named_directory', name) - #if doc is None: - #if path: - #doc = {'name': name, - #'path': path} - #self.dbs.add_document('named_directory', doc) - #else: - #if path: - #self.dbs.set_value('named_directory', name, 'path', path) - #else: - #self.dbs.remove_document('named_directory', name) - with self.db as dbs: - doc = dbs.get_document('named_directory', name) - if doc is None: - if path: - doc = {'name': name, - 'path': path} - dbs.add_document('named_directory', doc) + with self.storage.data(write=True) as db: + if path: + db.named_directory[name] = path else: - if path: - dbs.set_value('named_directory', name, 'path', path) - else: - dbs.remove_document('named_directory', name) + del db.named_directory[name] def named_directory(self, name): - #return self.dbs.get_value('named_directory', name, 'path') - with self.db as dbs: - return dbs.get_value('named_directory', name, 'path') + with self.storage.data() as db: + return db.named_directory[name].path.get() def named_directories(self): - #return self.dbs.filter_documents('named_directory', 'all') - with self.db as dbs: - return dbs.filter_documents('named_directory', 'all') + with self.storage.data() as db: + for row in db.named_directory.search(fields=["name"], as_list=True): + yield row[0] def set_json_value(self, name, json_value): - #doc = self.dbs.get_document('json_value', name) - #json_dict = {'value': json_value} - #if doc is None: - #doc = {'name': name, - #'json_dict': json_dict - #} - #self.dbs.add_document('json_value', doc) - #else: - #self.dbs.set_value('json_value', name, 'json_dict', json_dict) - with self.db as dbs: - doc = dbs.get_document('json_value', name) - json_dict = {'value': json_value} - if doc is None: - doc = {'name': name, - 'json_dict': json_dict - } - dbs.add_document('json_value', doc) - else: - dbs.set_value('json_value', name, 'json_dict', json_dict) + with self.storage.data(write=True) as db: + db["json_value"][name].json_dict = json_value def json_value(self, name): - #doc = self.dbs.get_document('json_value', name) - #if doc: - #return doc['json_dict']['value'] - #return None - with self.db as dbs: - doc = dbs.get_document('json_value', name) - if doc: - return doc['json_dict']['value'] - return None + with self.storage.data(write=True) as db: + return db["json_value"][name].json_dict.get() def set_path_metadata(self, path, metadata): - named_directory = metadata.get('named_directory') + named_directory = metadata.get("named_directory") if named_directory: - base_path = self.named_directory('capsul_engine') + base_path = self.named_directory("capsul_engine") if base_path: if not path.startswith(named_directory): - raise ValueError('Path "%s" is defined as relative to named directory %s but it does not start with "%s"' % (path, named_directory, base_path)) - path = path[len(base_path)+1:] + raise ValueError( + 'Path "%s" is defined as relative to named directory %s but it does not start with "%s"' + % (path, named_directory, base_path) + ) + path = path[len(base_path) + 1 :] else: if osp.isabs(path): - raise ValueError('Cannot determine relative path for "%s" because its base named directory "%s" is unknown' % (path, named_directory)) + raise ValueError( + 'Cannot determine relative path for "%s" because its base named directory "%s" is unknown' + % (path, named_directory) + ) else: if osp.isabs(path): for nd in self.named_directories(): if path.startswith(nd.path): named_directory = nd.name() - path = path[len(nd.path)+1] + path = path[len(nd.path) + 1] break else: named_directory = None else: # capsul_engine is the default named directory for relative paths - named_directory = 'capsul_engine' + named_directory = "capsul_engine" doc = metadata.copy() - doc['path'] = path + doc["path"] = path if named_directory: - doc['named_directory'] = named_directory + doc["named_directory"] = named_directory with self.db as dbs: - dbs.add_document('path_metadata', doc) - + dbs.add_document("path_metadata", doc) def path_metadata(self, path): if osp.isabs(path): for nd in self.named_directories(): if path.startswith(nd.path): - path = path[len(nd.path)+1:] + path = path[len(nd.path) + 1 :] break with self.db as dbs: - return dbs.get_document('path_metadata', path) + return dbs.get_document("path_metadata", path) diff --git a/capsul/engine/module/afni.py b/capsul/engine/module/afni.py index 60689293d..20bf0d39d 100644 --- a/capsul/engine/module/afni.py +++ b/capsul/engine/module/afni.py @@ -8,53 +8,58 @@ def init_settings(capsul_engine): with capsul_engine.settings as settings: - settings.ensure_module_fields('afni', - [dict(name='directory', - type='string', - description='Directory where AFNI is installed') - ]) + settings.ensure_module_fields( + "afni", + [ + dict( + name="directory", + type="str", + description="Directory where AFNI is installed", + ) + ], + ) # init a single config - config = settings.config('afni', 'global') + config = settings.config("afni", "global") if not config: - settings.new_config('afni', 'global', - {capsul_engine.settings.config_id_field: - 'afni'}) + settings.new_config( + "afni", "global", {capsul_engine.settings.config_id_field: "afni"} + ) def check_notably_invalid_config(conf): - ''' + """ Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled Returns ------- invalid: list list of invalid config keys - ''' + """ invalid = [] - for k in ('directory', ): - if getattr(conf, k, None) is None: + for k in ("directory",): + if conf.get(k) is None: invalid.append(k) return invalid def activate_configurations(): - ''' + """ Activate the AFNI module (set env variables) from the global configurations, in order to use them via :mod:`capsul.in_context.afni` functions - ''' - conf = engine.configurations.get('capsul.engine.module.afni', {}) - afni_dir = conf.get('directory') + """ + conf = engine.configurations.get("capsul.engine.module.afni", {}) + afni_dir = conf.get("directory") if afni_dir: - os.environ['AFNIPATH'] = six.ensure_str(afni_dir) - elif 'AFNIPATH' in os.environ: - del os.environ['AFNIPATH'] + os.environ["AFNIPATH"] = six.ensure_str(afni_dir) + elif "AFNIPATH" in os.environ: + del os.environ["AFNIPATH"] -def edition_widget(engine, environment, config_id='afni'): - ''' Edition GUI for AFNI config - see +def edition_widget(engine, environment, config_id="afni"): + """Edition GUI for AFNI config - see :class:`~capsul.qt_gui.widgets.settings_editor.SettingsEditor` - ''' + """ from soma.qt_gui.controller_widget import ScrollControllerWidget from soma.controller import Controller import types @@ -65,8 +70,8 @@ def validate_config(widget): controller = widget.controller_widget.controller with widget.engine.settings as session: conf = session.config(config_id, widget.environment) - values = {'config_id': config_id} - for k in ['directory']: + values = {"config_id": config_id} + for k in ["directory"]: value = getattr(controller, k) if value is traits.Undefined: value = None @@ -75,20 +80,21 @@ def validate_config(widget): session.new_config(config_id, widget.environment, values) else: for k, value in values.items(): - if k == 'config_id': + if k == "config_id": continue setattr(conf, k, values[k]) controller = Controller() - controller.add_trait('directory', traits.Directory(traits.Undefined, - desc='Directory where AFNI is installed')) + controller.add_trait( + "directory", + traits.Directory(traits.Undefined, desc="Directory where AFNI is installed"), + ) - conf = engine.settings.select_configurations( - environment, {'afni': 'any'}) + conf = engine.settings.select_configurations(environment, {"afni": "any"}) if conf: - fconf = conf.get('capsul.engine.module.afni', {}) - controller.directory = fconf.get('directory', traits.Undefined) + fconf = conf.get("capsul.engine.module.afni", {}) + controller.directory = fconf.get("directory", traits.Undefined) widget = ScrollControllerWidget(controller, live=True) widget.engine = engine diff --git a/capsul/engine/module/ants.py b/capsul/engine/module/ants.py index 976b64e73..d773f9913 100644 --- a/capsul/engine/module/ants.py +++ b/capsul/engine/module/ants.py @@ -8,53 +8,58 @@ def init_settings(capsul_engine): with capsul_engine.settings as settings: - settings.ensure_module_fields('ants', - [dict(name='directory', - type='string', - description='Directory where ANTS is installed') - ]) + settings.ensure_module_fields( + "ants", + [ + dict( + name="directory", + type="str", + description="Directory where ANTS is installed", + ) + ], + ) # init a single config - config = settings.config('ants', 'global') + config = settings.config("ants", "global") if not config: - settings.new_config('ants', 'global', - {capsul_engine.settings.config_id_field: - 'ants'}) + settings.new_config( + "ants", "global", {capsul_engine.settings.config_id_field: "ants"} + ) def check_notably_invalid_config(conf): - ''' + """ Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled Returns ------- invalid: list list of invalid config keys - ''' + """ invalid = [] - for k in ('directory', ): - if getattr(conf, k, None) is None: + for k in ("directory",): + if conf.get(k) is None: invalid.append(k) return invalid def activate_configurations(): - ''' + """ Activate the ANTS module (set env variables) from the global configurations, in order to use them via :mod:`capsul.in_context.ants` functions - ''' - conf = engine.configurations.get('capsul.engine.module.ants', {}) - ants_dir = conf.get('directory') + """ + conf = engine.configurations.get("capsul.engine.module.ants", {}) + ants_dir = conf.get("directory") if ants_dir: - os.environ['ANTSPATH'] = six.ensure_str(ants_dir) - elif 'ANTSPATH' in os.environ: - del os.environ['ANTSPATH'] + os.environ["ANTSPATH"] = six.ensure_str(ants_dir) + elif "ANTSPATH" in os.environ: + del os.environ["ANTSPATH"] -def edition_widget(engine, environment, config_id='ants'): - ''' Edition GUI for ANTS config - see +def edition_widget(engine, environment, config_id="ants"): + """Edition GUI for ANTS config - see :class:`~capsul.qt_gui.widgets.settings_editor.SettingsEditor` - ''' + """ from soma.qt_gui.controller_widget import ScrollControllerWidget from soma.controller import Controller import types @@ -65,8 +70,8 @@ def validate_config(widget): controller = widget.controller_widget.controller with widget.engine.settings as session: conf = session.config(config_id, widget.environment) - values = {'config_id': config_id} - for k in ['directory']: + values = {"config_id": config_id} + for k in ["directory"]: value = getattr(controller, k) if value is traits.Undefined: value = None @@ -75,20 +80,21 @@ def validate_config(widget): session.new_config(config_id, widget.environment, values) else: for k, value in values.items(): - if k == 'config_id': + if k == "config_id": continue setattr(conf, k, values[k]) controller = Controller() - controller.add_trait('directory', traits.Directory(traits.Undefined, - desc='Directory where ANTS is installed')) + controller.add_trait( + "directory", + traits.Directory(traits.Undefined, desc="Directory where ANTS is installed"), + ) - conf = engine.settings.select_configurations( - environment, {'ants': 'any'}) + conf = engine.settings.select_configurations(environment, {"ants": "any"}) if conf: - fconf = conf.get('capsul.engine.module.ants', {}) - controller.directory = fconf.get('directory', traits.Undefined) + fconf = conf.get("capsul.engine.module.ants", {}) + controller.directory = fconf.get("directory", traits.Undefined) widget = ScrollControllerWidget(controller, live=True) widget.engine = engine diff --git a/capsul/engine/module/attributes.py b/capsul/engine/module/attributes.py index bdfbc9949..5510df56c 100644 --- a/capsul/engine/module/attributes.py +++ b/capsul/engine/module/attributes.py @@ -26,16 +26,16 @@ def init_settings(capsul_engine): with capsul_engine.settings as session: session.ensure_module_fields('attributes', [dict(name='attributes_schema_paths', - type='list_string', + type='list[str]', description='attributes shchemas modules names'), dict(name='attributes_schemas', - type='json', + type='dict', description='attributes shchemas names'), dict(name='process_completion', - type='string', + type='str', description='process completion model name'), dict(name='path_completion', - type='string', + type='str', description='path completion model name'), ]) config = session.config('attributes', 'global') diff --git a/capsul/engine/module/axon.py b/capsul/engine/module/axon.py index eeaa53a1d..5398fad14 100644 --- a/capsul/engine/module/axon.py +++ b/capsul/engine/module/axon.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -''' +""" Configuration module which links with `Axon `_ -''' +""" from __future__ import absolute_import import os @@ -13,88 +13,99 @@ def init_settings(capsul_engine): with capsul_engine.settings as settings: - settings.ensure_module_fields('axon', - [dict(name='shared_directory', - type='string', - description= - 'Directory where BrainVisa shared data is installed'), - dict(name='user_level', - type='int', - description= - '0: basic, 1: advanced, 2: expert, or more. ' - 'used to display or hide some advanced features or ' - 'process parameters that would be confusing to a novice ' - 'user'), - ]) + settings.ensure_module_fields( + "axon", + [ + dict( + name="shared_directory", + type="str", + description="Directory where BrainVisa shared data is installed", + ), + dict( + name="user_level", + type="int", + description="0: basic, 1: advanced, 2: expert, or more. " + "used to display or hide some advanced features or " + "process parameters that would be confusing to a novice " + "user", + ), + ], + ) with capsul_engine.settings as session: - config = session.config('axon', 'global') + config = session.config("axon", "global") if not config: from soma import config as soma_config + shared_dir = soma_config.BRAINVISA_SHARE - values = {capsul_engine.settings.config_id_field: 'axon', - 'shared_directory': shared_dir, 'user_level': 0} - session.new_config('axon', 'global', values) + values = { + capsul_engine.settings.config_id_field: "axon", + "shared_directory": shared_dir, + "user_level": 0, + } + session.new_config("axon", "global", values) # link with StudyConfig - if hasattr(capsul_engine, 'study_config'): - if 'BrainVISAConfig' not in capsul_engine.study_config.modules: - scmod = capsul_engine.study_config.load_module( - 'BrainVISAConfig', {}) + if hasattr(capsul_engine, "study_config"): + if "BrainVISAConfig" not in capsul_engine.study_config.modules: + scmod = capsul_engine.study_config.load_module("BrainVISAConfig", {}) scmod.initialize_module() scmod.initialize_callbacks() else: - scmod = capsul_engine.study_config.modules['BrainVISAConfig'] + scmod = capsul_engine.study_config.modules["BrainVISAConfig"] scmod.sync_to_engine() + def check_configurations(): - ''' + """ Checks if the activated configuration is valid to use BrainVisa and returns an error message if there is an error or None if everything is good. - ''' - shared_dir = capsul.engine.configurations.get( - 'axon', {}).get('shared_directory', '') + """ + shared_dir = capsul.engine.configurations.get("axon", {}).get( + "shared_directory", "" + ) if not shared_dir: - return 'Axon shared_directory is not found' + return "Axon shared_directory is not found" return None def check_notably_invalid_config(conf): - ''' + """ Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled Returns ------- invalid: list list of invalid config keys - ''' + """ invalid = [] - for k in ('shared_directory', ): - if getattr(conf, k, None) is None: + for k in ("shared_directory",): + if conf.get(k) is None: invalid.append(k) return invalid def complete_configurations(): - ''' + """ Try to automatically set or complete the capsul.engine.configurations for Axon. - ''' + """ config = capsul.engine.configurations - config = config.setdefault('axon', {}) - shared_dir = config.get('shared_directory', None) + config = config.setdefault("axon", {}) + shared_dir = config.get("shared_directory", None) if shared_dir is None: from soma import config as soma_config + shared_dir = soma_config.BRAINVISA_SHARE if shared_dir: - config['shared_directory'] = shared_dir + config["shared_directory"] = shared_dir -def edition_widget(engine, environment, config_id='axon'): - ''' Edition GUI for axon config - see +def edition_widget(engine, environment, config_id="axon"): + """Edition GUI for axon config - see :class:`~capsul.qt_gui.widgets.settings_editor.SettingsEditor` - ''' + """ from soma.qt_gui.controller_widget import ScrollControllerWidget from soma.controller import Controller import types @@ -105,38 +116,40 @@ def validate_config(widget): controller = widget.controller_widget.controller with widget.engine.settings as session: conf = session.config(config_id, widget.environment) - values = {'config_id': config_id, - 'user_level': controller.user_level} - if controller.shared_directory in (None, traits.Undefined, ''): - values['shared_directory'] = None + values = {"config_id": config_id, "user_level": controller.user_level} + if controller.shared_directory in (None, traits.Undefined, ""): + values["shared_directory"] = None else: - values['shared_directory'] = controller.shared_directory + values["shared_directory"] = controller.shared_directory if conf is None: session.new_config(config_id, widget.environment, values) else: - for k in ('shared_directory', 'user_level'): + for k in ("shared_directory", "user_level"): setattr(conf, k, values[k]) controller = Controller() - controller.add_trait('shared_directory', - traits.Directory(desc='Directory where BrainVisa ' - 'shared data is installed')) controller.add_trait( - 'user_level', - traits.Int(desc= - '0: basic, 1: advanced, 2: expert, or more. ' - 'used to display or hide some advanced features or ' - 'process parameters that would be confusing to a novice ' - 'user')) - - conf = engine.settings.select_configurations( - environment, {'axon': 'any'}) + "shared_directory", + traits.Directory(desc="Directory where BrainVisa " "shared data is installed"), + ) + controller.add_trait( + "user_level", + traits.Int( + desc="0: basic, 1: advanced, 2: expert, or more. " + "used to display or hide some advanced features or " + "process parameters that would be confusing to a novice " + "user" + ), + ) + + conf = engine.settings.select_configurations(environment, {"axon": "any"}) if conf: - controller.shared_directory = conf.get( - 'capsul.engine.module.axon', {}).get('shared_directory', - traits.Undefined) - controller.user_level = conf.get( - 'capsul.engine.module.axon', {}).get('user_level', 0) + controller.shared_directory = conf.get("capsul.engine.module.axon", {}).get( + "shared_directory", traits.Undefined + ) + controller.user_level = conf.get("capsul.engine.module.axon", {}).get( + "user_level", 0 + ) widget = ScrollControllerWidget(controller, live=True) widget.engine = engine diff --git a/capsul/engine/module/fom.py b/capsul/engine/module/fom.py index 745d6e837..a91c66bc2 100644 --- a/capsul/engine/module/fom.py +++ b/capsul/engine/module/fom.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -''' +""" Config module for :mod:`File Organization models (FOMs) ` Classes ======= :class:`FomConfig` ------------------ -''' +""" from __future__ import absolute_import import os @@ -22,94 +22,121 @@ def init_settings(capsul_engine): with capsul_engine.settings as session: - session.ensure_module_fields('fom', - [dict(name='input_fom', type='string', description='input FOM'), - dict(name='output_fom', type='string', description='output FOM'), - dict(name='shared_fom', type='string', - description='shared data FOM'), - dict(name='volumes_format', type='string', - description='Format used for volumes'), - dict(name='meshes_format', type='string', - description='Format used for meshes'), - dict(name='auto_fom', type='boolean', - description='Look in all FOMs when a process is not found ' - '(in addition to the standard share/foms). Note that ' - 'auto_fom looks for the first FOM matching the process to ' - 'get completion for, and does not handle ambiguities. ' - 'Moreover it brings an overhead (typically 6-7 seconds) the ' - 'first time it is used since it has to parse all available ' - 'FOMs.'), - dict(name='fom_path', type='list_string', - description='list of additional directories where to look ' - 'for FOMs'), - # FIXME: until directories are included in another config module - dict(name='input_directory', type='string', - description='input study data directory'), - dict(name='output_directory', type='string', - description='output study data directory'), - ]) - - capsul_engine.load_module('capsul.engine.module.axon') - capsul_engine.load_module('capsul.engine.module.spm') - capsul_engine.load_module('capsul.engine.module.attributes') + session.ensure_module_fields( + "fom", + [ + dict(name="input_fom", type="str", description="input FOM"), + dict(name="output_fom", type="str", description="output FOM"), + dict(name="shared_fom", type="str", description="shared data FOM"), + dict( + name="volumes_format", + type="str", + description="Format used for volumes", + ), + dict( + name="meshes_format", + type="str", + description="Format used for meshes", + ), + dict( + name="auto_fom", + type="bool", + description="Look in all FOMs when a process is not found " + "(in addition to the standard share/foms). Note that " + "auto_fom looks for the first FOM matching the process to " + "get completion for, and does not handle ambiguities. " + "Moreover it brings an overhead (typically 6-7 seconds) the " + "first time it is used since it has to parse all available " + "FOMs.", + ), + dict( + name="fom_path", + type="list[str]", + description="list of additional directories where to look " + "for FOMs", + ), + # FIXME: until directories are included in another config module + dict( + name="input_directory", + type="str", + description="input study data directory", + ), + dict( + name="output_directory", + type="str", + description="output study data directory", + ), + ], + ) + + capsul_engine.load_module("capsul.engine.module.axon") + capsul_engine.load_module("capsul.engine.module.spm") + capsul_engine.load_module("capsul.engine.module.attributes") with capsul_engine.settings as session: - config = session.config('fom', 'global') + config = session.config("fom", "global") if not config: - values = {capsul_engine.settings.config_id_field: 'fom', - 'auto_fom': True, 'fom_path': []} - session.new_config('fom', 'global', values) - - if not hasattr(capsul_engine, '_modules_data'): + values = { + capsul_engine.settings.config_id_field: "fom", + "auto_fom": True, + "fom_path": [], + } + session.new_config("fom", "global", values) + + if not hasattr(capsul_engine, "_modules_data"): capsul_engine._modules_data = {} - store = capsul_engine._modules_data.setdefault('fom', {}) - store['foms'] = {} - store['all_foms'] = SortedDictionary() - store['fom_atp'] = {'all': {}} - store['fom_pta'] = {'all': {}} - - capsul_engine.settings.module_notifiers['capsul.engine.module.fom'] \ - = [partial(fom_config_updated, weakref.proxy(capsul_engine), 'global')] + store = capsul_engine._modules_data.setdefault("fom", {}) + store["foms"] = {} + store["all_foms"] = SortedDictionary() + store["fom_atp"] = {"all": {}} + store["fom_pta"] = {"all": {}} + + capsul_engine.settings.module_notifiers["capsul.engine.module.fom"] = [ + partial(fom_config_updated, weakref.proxy(capsul_engine), "global") + ] capsul_engine.settings.module_notifiers.setdefault( - 'capsul.engine.module.axon', []).append( - partial(config_updated, weakref.proxy(capsul_engine), 'global')) + "capsul.engine.module.axon", [] + ).append(partial(config_updated, weakref.proxy(capsul_engine), "global")) capsul_engine.settings.module_notifiers.setdefault( - 'capsul.engine.module.spm', []).append( - partial(config_updated, weakref.proxy(capsul_engine), 'global')) + "capsul.engine.module.spm", [] + ).append(partial(config_updated, weakref.proxy(capsul_engine), "global")) # link with StudyConfig - if hasattr(capsul_engine, 'study_config') \ - and 'FomConfig' not in capsul_engine.study_config.modules: - scmod = capsul_engine.study_config.load_module('FomConfig', {}) + if ( + hasattr(capsul_engine, "study_config") + and "FomConfig" not in capsul_engine.study_config.modules + ): + scmod = capsul_engine.study_config.load_module("FomConfig", {}) scmod.initialize_module() scmod.initialize_callbacks() - update_fom(capsul_engine, 'global') + update_fom(capsul_engine, "global") def config_dependencies(config): - return {'axon': 'any', - #'spm': 'any', - 'attributes': 'any'} + return { + "axon": "any", + #'spm': 'any', + "attributes": "any", + } def config_updated(capsul_engine, environment, param=None, value=None): - if param in (None, 'directory', 'shared_directory'): + if param in (None, "directory", "shared_directory"): update_fom(capsul_engine, environment) def spm_config_updated(capsul_engine, environment, param=None, value=None): - if param in (None, 'directory'): + if param in (None, "directory"): update_fom(capsul_engine, environment) -def fom_config_updated(capsul_engine, environment='global', - param=None, value=None): - if param in ('volumes_format', 'meshes_format'): +def fom_config_updated(capsul_engine, environment="global", param=None, value=None): + if param in ("volumes_format", "meshes_format"): update_formats(capsul_engine, environment) return - if param in ('fom_path', ): + if param in ("fom_path",): reset_foms(capsul_engine, environment) return @@ -117,100 +144,103 @@ def fom_config_updated(capsul_engine, environment='global', update_fom(capsul_engine, environment, param, value) -def update_fom(capsul_engine, environment='global', param=None, value=None): - '''Load configured FOMs and create FOM completion data - ''' - #print('***update_fom ***') +def update_fom(capsul_engine, environment="global", param=None, value=None): + """Load configured FOMs and create FOM completion data""" + # print('***update_fom ***') with capsul_engine.settings as session: - config = session.config('fom', environment) + config = session.config("fom", environment) if config is None: return - soma_app = Application('capsul', plugin_modules=['soma.fom']) - if 'soma.fom' not in soma_app.loaded_plugin_modules: + soma_app = Application("capsul", plugin_modules=["soma.fom"]) + if "soma.fom" not in soma_app.loaded_plugin_modules: # WARNING: this is unsafe, may erase configured things, and # probably not thread-safe. soma_app.initialize() if config.fom_path: - fom_path = [p for p in config.fom_path - if p not in soma_app.fom_path] \ - + soma_app.fom_path + fom_path = [ + p for p in config.fom_path if p not in soma_app.fom_path + ] + soma_app.fom_path else: fom_path = list(soma_app.fom_path) soma_app.fom_manager.paths = fom_path soma_app.fom_manager.fom_files() - store = capsul_engine._modules_data['fom'] - if config.auto_fom \ - and len(store['all_foms']) <= 3: + store = capsul_engine._modules_data["fom"] + if config.auto_fom and len(store["all_foms"]) <= 3: for schema in soma_app.fom_manager.fom_files(): - if schema not in store['all_foms']: - store['all_foms'][schema] = None # not loaded yet. - - foms = (('input', config.input_fom), - ('output', config.output_fom), - ('shared', config.shared_fom)) + if schema not in store["all_foms"]: + store["all_foms"][schema] = None # not loaded yet. + + foms = ( + ("input", config.input_fom), + ("output", config.output_fom), + ("shared", config.shared_fom), + ) for fom_type, fom_filename in foms: if fom_filename not in ("", None, traits.Undefined): - fom = store['all_foms'].get(fom_filename) + fom = store["all_foms"].get(fom_filename) if fom is None: - fom, atp, pta = load_fom(capsul_engine, fom_filename, - config, session, environment) + fom, atp, pta = load_fom( + capsul_engine, fom_filename, config, session, environment + ) else: - atp = store['fom_atp']['all'][fom_filename] - pta = store['fom_pta']['all'][fom_filename] - store['foms'][fom_type] = fom - store['fom_atp'][fom_type] = atp - store['fom_pta'][fom_type] = pta + atp = store["fom_atp"]["all"][fom_filename] + pta = store["fom_pta"]["all"][fom_filename] + store["foms"][fom_type] = fom + store["fom_atp"][fom_type] = atp + store["fom_pta"][fom_type] = pta # update directories directories = {} - spm = session.config('spm', environment) + spm = session.config("spm", environment) if spm: - directories['spm'] = spm.directory - axon = session.config('axon', environment) + directories["spm"] = spm.directory + axon = session.config("axon", environment) if axon: - directories['shared'] = axon.shared_directory - directories['input'] = config.input_directory - directories['output'] = config.output_directory + directories["shared"] = axon.shared_directory + directories["input"] = config.input_directory + directories["output"] = config.output_directory - for atp in store['fom_atp']['all'].values(): + for atp in store["fom_atp"]["all"].values(): atp.directories = directories # backward compatibility for StudyConfig - capsul_engine.study_config.modules_data.foms = store['foms'] - capsul_engine.study_config.modules_data.all_foms = store['all_foms'] - capsul_engine.study_config.modules_data.fom_atp = store['fom_atp'] - capsul_engine.study_config.modules_data.fom_pta = store['fom_pta'] + capsul_engine.study_config.modules_data.foms = store["foms"] + capsul_engine.study_config.modules_data.all_foms = store["all_foms"] + capsul_engine.study_config.modules_data.fom_atp = store["fom_atp"] + capsul_engine.study_config.modules_data.fom_pta = store["fom_pta"] def update_formats(capsul_engine, environment): with capsul_engine.settings as session: - config = session.config('fom', environment) + config = session.config("fom", environment) if config is None: return directories = {} - spm = session.config('spm', environment) + spm = session.config("spm", environment) if spm: - directories['spm'] = spm.directory - axon = session.config('axon', environment) + directories["spm"] = spm.directory + axon = session.config("axon", environment) if axon: - directories['shared'] = axon.shared_directory - directories['input'] = config.input_directory - directories['output'] = config.output_directory + directories["shared"] = axon.shared_directory + directories["input"] = config.input_directory + directories["output"] = config.output_directory - fields = config._dbs.get_fields_names(config._collection) - formats = tuple(getattr(config, key) \ - for key in fields \ - if key.endswith('_format') \ - and getattr(config, key) is not None) + with config._storage.data() as data: + fields = data[config._collection].keys() + formats = tuple( + getattr(config, key) + for key in fields + if key.endswith("_format") and getattr(config, key) is not None + ) - store = capsul_engine._modules_data['fom'] + store = capsul_engine._modules_data["fom"] - for schema, fom in store['all_foms'].items(): + for schema, fom in store["all_foms"].items(): if fom is None: continue @@ -218,68 +248,72 @@ def update_formats(capsul_engine, environment): fom, selection={}, directories=directories, - preferred_formats=set((formats))) - old_atp = store['fom_atp']['all'].get(schema) - store['fom_atp']['all'][schema] = atp + preferred_formats=set((formats)), + ) + old_atp = store["fom_atp"]["all"].get(schema) + store["fom_atp"]["all"][schema] = atp if old_atp is not None: - for t in ('input', 'output', 'shared'): - if store['fom_atp'].get(t) is old_atp: - store['fom_atp'][t] = atp + for t in ("input", "output", "shared"): + if store["fom_atp"].get(t) is old_atp: + store["fom_atp"][t] = atp -def load_fom(capsul_engine, schema, config, session, environment='global'): - #print('=== load fom', schema, '===') - #import time - #t0 = time.time() - soma_app = Application('capsul', plugin_modules=['soma.fom']) - if 'soma.fom' not in soma_app.loaded_plugin_modules: +def load_fom(capsul_engine, schema, config, session, environment="global"): + # print('=== load fom', schema, '===') + # import time + # t0 = time.time() + soma_app = Application("capsul", plugin_modules=["soma.fom"]) + if "soma.fom" not in soma_app.loaded_plugin_modules: # WARNING: this is unsafe, may erase configured things, and # probably not thread-safe. soma_app.initialize() old_fom_path = soma_app.fom_path if config.fom_path: - soma_app.fom_path = [six.ensure_str(p) for p in config.fom_path - if p not in soma_app.fom_path] \ - + soma_app.fom_path + soma_app.fom_path = [ + six.ensure_str(p) for p in config.fom_path if p not in soma_app.fom_path + ] + soma_app.fom_path else: soma_app.fom_path = list(soma_app.fom_path) - fom = soma_app.fom_manager.load_foms(schema) + try: + fom = soma_app.fom_manager.load_foms(schema) + except KeyError: + return None, None, None soma_app.fom_path = old_fom_path - store = capsul_engine._modules_data['fom'] - store['all_foms'][schema] = fom + store = capsul_engine._modules_data["fom"] + store["all_foms"][schema] = fom # Create FOM completion data - fields = config._dbs.get_fields_names(config._collection) - formats = tuple(getattr(config, key) \ - for key in fields \ - if key.endswith('_format') \ - and getattr(config, key) is not None) + with config._storage.data() as data: + fields = data[config._collection].keys() + formats = tuple( + getattr(config, key) + for key in fields + if key.endswith("_format") and getattr(config, key) is not None + ) directories = {} - spm = session.config('spm', environment) + spm = session.config("spm", environment) if spm: - directories['spm'] = spm.directory - axon = session.config('axon', environment) + directories["spm"] = spm.directory + axon = session.config("axon", environment) if axon: - directories['shared'] = axon.shared_directory - directories['input'] = config.input_directory - directories['output'] = config.output_directory + directories["shared"] = axon.shared_directory + directories["input"] = config.input_directory + directories["output"] = config.output_directory atp = AttributesToPaths( - fom, - selection={}, - directories=directories, - preferred_formats=set((formats))) - store['fom_atp']['all'][schema] = atp + fom, selection={}, directories=directories, preferred_formats=set((formats)) + ) + store["fom_atp"]["all"][schema] = atp pta = PathToAttributes(fom, selection={}) - store['fom_pta']['all'][schema] = pta - #print(' load fom done:', time.time() - t0, 's') + store["fom_pta"]["all"][schema] = pta + # print(' load fom done:', time.time() - t0, 's') return fom, atp, pta def reset_foms(capsul_engine, environment): - soma_app = Application('capsul', plugin_modules=['soma.fom']) - if 'soma.fom' not in soma_app.loaded_plugin_modules: + soma_app = Application("capsul", plugin_modules=["soma.fom"]) + if "soma.fom" not in soma_app.loaded_plugin_modules: # WARNING: this is unsafe, may erase configured things, and # probably not thread-safe. soma_app.initialize() @@ -287,10 +321,10 @@ def reset_foms(capsul_engine, environment): update_fom(capsul_engine, environment) -def edition_widget(engine, environment, config_id='fom'): - ''' Edition GUI for FOM config - see +def edition_widget(engine, environment, config_id="fom"): + """Edition GUI for FOM config - see :class:`~capsul.qt_gui.widgets.settings_editor.SettingsEditor` - ''' + """ from soma.qt_gui.controller_widget import ScrollControllerWidget from soma.controller import Controller import types @@ -300,13 +334,21 @@ def validate_config(widget): controller = widget.controller_widget.controller with widget.engine.settings as session: conf = session.config(config_id, widget.environment) - values = {'config_id': config_id} - for k in ('input_fom', 'output_fom', 'shared_fom', - 'volumes_format', 'meshes_format', 'auto_fom', - 'fom_path', 'input_directory', 'output_directory'): + values = {"config_id": config_id} + for k in ( + "input_fom", + "output_fom", + "shared_fom", + "volumes_format", + "meshes_format", + "auto_fom", + "fom_path", + "input_directory", + "output_directory", + ): value = getattr(controller, k) if value is traits.Undefined: - if k in ('fom_path', ): + if k in ("fom_path",): value = [] else: value = None @@ -315,74 +357,75 @@ def validate_config(widget): session.new_config(config_id, widget.environment, values) else: for k, value in values.items(): - if k == 'config_id': + if k == "config_id": continue setattr(conf, k, values[k]) controller = Controller() controller.add_trait( - 'input_fom', - traits.Str(traits.Undefined, output=False, desc='input FOM')) + "input_fom", traits.Str(traits.Undefined, output=False, desc="input FOM") + ) controller.add_trait( - 'output_fom', - traits.Str(traits.Undefined, output=False, desc='output FOM')) + "output_fom", traits.Str(traits.Undefined, output=False, desc="output FOM") + ) controller.add_trait( - 'shared_fom', - traits.Str(traits.Undefined, output=False, desc='shared data FOM')) + "shared_fom", traits.Str(traits.Undefined, output=False, desc="shared data FOM") + ) controller.add_trait( - 'volumes_format', - traits.Str(traits.Undefined, output=False, desc='Format used for volumes')) + "volumes_format", + traits.Str(traits.Undefined, output=False, desc="Format used for volumes"), + ) controller.add_trait( - 'meshes_format', - traits.Str(traits.Undefined, output=False, - desc='Format used for meshes')) + "meshes_format", + traits.Str(traits.Undefined, output=False, desc="Format used for meshes"), + ) controller.add_trait( - 'auto_fom', - traits.Bool(True, output=False, - desc='Look in all FOMs when a process is not found (in ' - 'addition to the standard share/foms). Note that auto_fom ' - 'looks for the first FOM matching the process to get ' - 'completion for, and does not handle ambiguities. Moreover ' - 'it brings an overhead (typically 6-7 seconds) the first ' - 'time it is used since it has to parse all available FOMs.')) + "auto_fom", + traits.Bool( + True, + output=False, + desc="Look in all FOMs when a process is not found (in " + "addition to the standard share/foms). Note that auto_fom " + "looks for the first FOM matching the process to get " + "completion for, and does not handle ambiguities. Moreover " + "it brings an overhead (typically 6-7 seconds) the first " + "time it is used since it has to parse all available FOMs.", + ), + ) controller.add_trait( - 'fom_path', - traits.List(traits.Directory(output=False), - desc='list of additional directories where to look for FOMs')) + "fom_path", + traits.List( + traits.Directory(output=False), + desc="list of additional directories where to look for FOMs", + ), + ) # FIXME: until directories are included in another config module controller.add_trait( - 'input_directory', - traits.Directory(traits.Undefined, output=False, - desc='input study data directory')) + "input_directory", + traits.Directory( + traits.Undefined, output=False, desc="input study data directory" + ), + ) controller.add_trait( - 'output_directory', - traits.Directory(traits.Undefined, output=False, - desc='output study data directory')) + "output_directory", + traits.Directory( + traits.Undefined, output=False, desc="output study data directory" + ), + ) - conf = engine.settings.select_configurations( - environment, {'fom': 'any'}) + conf = engine.settings.select_configurations(environment, {"fom": "any"}) if conf: - fconf = conf.get( - 'capsul.engine.module.fom', {}) - controller.input_fom = fconf.get( - 'input_fom', traits.Undefined) - controller.output_fom = fconf.get( - 'output_fom', traits.Undefined) - controller.shared_fom = fconf.get( - 'shared_fom', traits.Undefined) - controller.volumes_format= fconf.get( - 'volumes_format', traits.Undefined) - controller.meshes_format = fconf.get( - 'meshes_format', traits.Undefined) - controller.auto_fom = fconf.get( - 'auto_fom', traits.Undefined) - controller.fom_path = fconf.get( - 'fom_path', traits.Undefined) - controller.input_directory= fconf.get( - 'input_directory', traits.Undefined) - controller.output_directory = fconf.get( - 'output_directory', traits.Undefined) + fconf = conf.get("capsul.engine.module.fom", {}) + controller.input_fom = fconf.get("input_fom", traits.Undefined) + controller.output_fom = fconf.get("output_fom", traits.Undefined) + controller.shared_fom = fconf.get("shared_fom", traits.Undefined) + controller.volumes_format = fconf.get("volumes_format", traits.Undefined) + controller.meshes_format = fconf.get("meshes_format", traits.Undefined) + controller.auto_fom = fconf.get("auto_fom", traits.Undefined) + controller.fom_path = fconf.get("fom_path", traits.Undefined) + controller.input_directory = fconf.get("input_directory", traits.Undefined) + controller.output_directory = fconf.get("output_directory", traits.Undefined) widget = ScrollControllerWidget(controller, live=True) widget.engine = engine diff --git a/capsul/engine/module/freesurfer.py b/capsul/engine/module/freesurfer.py index 6b5e1ec01..774e30b70 100644 --- a/capsul/engine/module/freesurfer.py +++ b/capsul/engine/module/freesurfer.py @@ -10,87 +10,100 @@ def init_settings(capsul_engine): with capsul_engine.settings as settings: - settings.ensure_module_fields('freesurfer', - [dict(name='setup', - type='string', - description='path of FreeSurferEnv.sh file'), - dict(name='subjects_dir', - type='string', - description='Freesurfer subjects data directory'), - ]) + settings.ensure_module_fields( + "freesurfer", + [ + dict( + name="setup", + type="str", + description="path of FreeSurferEnv.sh file", + ), + dict( + name="subjects_dir", + type="str", + description="Freesurfer subjects data directory", + ), + ], + ) # init a single config - config = settings.config('freesurfer', 'global') + config = settings.config("freesurfer", "global") if not config: - settings.new_config('freesurfer', 'global', - {capsul_engine.settings.config_id_field: - 'freesurfer'}) + settings.new_config( + "freesurfer", + "global", + {capsul_engine.settings.config_id_field: "freesurfer"}, + ) # link with StudyConfig - if hasattr(capsul_engine, 'study_config') \ - and 'FreeSurferConfig' not in capsul_engine.study_config.modules: - fsmod = capsul_engine.study_config.load_module('FreeSurferConfig', {}) + if ( + hasattr(capsul_engine, "study_config") + and "FreeSurferConfig" not in capsul_engine.study_config.modules + ): + fsmod = capsul_engine.study_config.load_module("FreeSurferConfig", {}) fsmod.initialize_module() fsmod.initialize_callbacks() def check_configurations(): - ''' + """ Checks if the activated configuration is valid to run Freesurfer and returns an error message if there is an error or None if everything is good. - ''' + """ fs_setup = capsul.engine.configurations.get( - 'capsul.engine.module.freesurfer', {}).get('setup') + "capsul.engine.module.freesurfer", {} + ).get("setup") if not fs_setup: - return 'Freesurfer setup script FreeSurferEnv.sh is not configured' + return "Freesurfer setup script FreeSurferEnv.sh is not configured" return None def check_notably_invalid_config(conf): - ''' + """ Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled Returns ------- invalid: list list of invalid config keys - ''' + """ invalid = [] - for k in ('setup', 'subjects_dir'): - if getattr(conf, k, None) is None: + for k in ("setup", "subjects_dir"): + if conf.get(k) is None: invalid.append(k) return invalid def complete_configurations(): - ''' + """ Try to automatically set or complete the capsul.engine.configurations for Freesurfer. - ''' + """ config = capsul.engine.configurations - config = config.setdefault('freesurfer', {}) - fs_setup = config.get('setup') + config = config.setdefault("freesurfer", {}) + fs_setup = config.get("setup") if not fs_setup: - fs_home = os.environ.get('FREESURFER_HOME') + fs_home = os.environ.get("FREESURFER_HOME") if fs_home: - fs_setup = osp.join(fs_home, 'FreeSurferEnv.sh') + fs_setup = osp.join(fs_home, "FreeSurferEnv.sh") if not osp.exists(fs_setup): fs_setup = None if not fs_setup: - reconall = find_in_path('recon-all') + reconall = find_in_path("recon-all") if reconall: - fs_setup = osp.join(osp.dirname(osp.dirname(reconall)), - 'FreeSurferEnv.sh') + fs_setup = osp.join( + osp.dirname(osp.dirname(reconall)), "FreeSurferEnv.sh" + ) if not osp.exists(fs_setup): fs_setup = None if fs_setup: - config['setup'] = fs_setup + config["setup"] = fs_setup -def edition_widget(engine, environment, config_id='freesurfer'): - ''' Edition GUI for Freesurfer config - see +def edition_widget(engine, environment, config_id="freesurfer"): + """Edition GUI for Freesurfer config - see :class:`~capsul.qt_gui.widgets.settings_editor.SettingsEditor` - ''' + """ from soma.qt_gui.controller_widget import ScrollControllerWidget from soma.controller import Controller import types @@ -101,8 +114,8 @@ def validate_config(widget): controller = widget.controller_widget.controller with widget.engine.settings as session: conf = session.config(config_id, widget.environment) - values = {'config_id': config_id} - for k in ('setup', 'subjects_dir'): + values = {"config_id": config_id} + for k in ("setup", "subjects_dir"): value = getattr(controller, k) if value is traits.Undefined: value = None @@ -111,28 +124,28 @@ def validate_config(widget): session.new_config(config_id, widget.environment, values) else: for k, value in values.items(): - if k == 'config_id': + if k == "config_id": continue setattr(conf, k, values[k]) controller = Controller() - controller.add_trait("setup", traits.File( - traits.Undefined, - desc="Path to 'FreeSurferEnv.sh'")) - controller.add_trait('subjectsdir', traits.Directory( - traits.Undefined, - desc='FreeSurfer subjects data directory')) + controller.add_trait( + "setup", traits.File(traits.Undefined, desc="Path to 'FreeSurferEnv.sh'") + ) + controller.add_trait( + "subjectsdir", + traits.Directory(traits.Undefined, desc="FreeSurfer subjects data directory"), + ) - conf = engine.settings.select_configurations( - environment, {'freesurfer': 'any'}) + conf = engine.settings.select_configurations(environment, {"freesurfer": "any"}) if conf: - controller.setup = conf.get( - 'capsul.engine.module.freesurfer', {}).get('setup', - traits.Undefined) - controller.subjects_dir= conf.get( - 'capsul.engine.module.freesurfer', {}).get('subjects_dir', - traits.Undefined) + controller.setup = conf.get("capsul.engine.module.freesurfer", {}).get( + "setup", traits.Undefined + ) + controller.subjects_dir = conf.get("capsul.engine.module.freesurfer", {}).get( + "subjects_dir", traits.Undefined + ) widget = ScrollControllerWidget(controller, live=True) widget.engine = engine diff --git a/capsul/engine/module/fsl.py b/capsul/engine/module/fsl.py index 6a241f8a6..12de6aebc 100644 --- a/capsul/engine/module/fsl.py +++ b/capsul/engine/module/fsl.py @@ -11,111 +11,118 @@ def init_settings(capsul_engine): with capsul_engine.settings as settings: - settings.ensure_module_fields('fsl', - [dict(name='directory', - type='string', - description='Directory where FSL is installed'), - dict(name='config', - type='string', - description='path of fsl.sh file'), - dict(name='prefix', - type='string', - description='Prefix to add to FSL commands') - ]) + settings.ensure_module_fields( + "fsl", + [ + dict( + name="directory", + type="str", + description="Directory where FSL is installed", + ), + dict(name="config", type="str", description="path of fsl.sh file"), + dict( + name="prefix", + type="str", + description="Prefix to add to FSL commands", + ), + ], + ) # init a single config - config = settings.config('fsl', 'global') + config = settings.config("fsl", "global") if not config: - settings.new_config('fsl', 'global', - {capsul_engine.settings.config_id_field: - 'fsl'}) + settings.new_config( + "fsl", "global", {capsul_engine.settings.config_id_field: "fsl"} + ) + - def check_configurations(): - ''' + """ Checks if the activated configuration is valid to run FSL and returns an error message if there is an error or None if everything is good. - ''' - fsl_prefix = capsul.engine.configurations.get( - 'capsul.engine.module.fsl',{}).get('prefix', '') - fsl_config = capsul.engine.configurations.get( - 'capsul.engine.module.fsl',{}).get('config') + """ + fsl_prefix = capsul.engine.configurations.get("capsul.engine.module.fsl", {}).get( + "prefix", "" + ) + fsl_config = capsul.engine.configurations.get("capsul.engine.module.fsl", {}).get( + "config" + ) if not fsl_config: - if not find_in_path('%sbet' % fsl_prefix): + if not find_in_path("%sbet" % fsl_prefix): return 'FSL command "%sbet" cannot be found in PATH' % fsl_prefix else: if fsl_prefix: - return 'FSL configuration must either use config or prefix but not both' + return "FSL configuration must either use config or prefix but not both" if not osp.exists(fsl_config): return 'File "%s" does not exist' % fsl_config - if not fsl_config.endswith('fsl.sh'): + if not fsl_config.endswith("fsl.sh"): return 'File "%s" is not a path to fsl.sh script' % fsl_config return None def check_notably_invalid_config(conf): - ''' + """ Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled Returns ------- invalid: list list of invalid config keys - ''' + """ invalid = [] - for k in ('directory', 'config'): - if getattr(conf, k, None) is None: + for k in ("directory", "config"): + if conf.get(k) is None: invalid.append(k) return invalid def complete_configurations(): - ''' + """ Try to automatically set or complete the capsul.engine.configurations for FSL. - ''' + """ config = capsul.engine.configurations - config = config.setdefault('capsul.engine.module.fsl', {}) - fsl_dir = config.get('directory', os.environ.get('FSLDIR')) - fsl_prefix = config.get('prefix', '') + config = config.setdefault("capsul.engine.module.fsl", {}) + fsl_dir = config.get("directory", os.environ.get("FSLDIR")) + fsl_prefix = config.get("prefix", "") if fsl_dir and not fsl_prefix: # Try to set fsl_config from FSLDIR environment variable - fsl_config = '%s/etc/fslconf/fsl.sh' % fsl_dir + fsl_config = "%s/etc/fslconf/fsl.sh" % fsl_dir if osp.exists(fsl_config): - config['config'] = fsl_config + config["config"] = fsl_config elif not fsl_prefix: # Try to set fsl_prefix by searching fsl-*bet in PATH - bet = find_in_path('fsl*-bet') + bet = find_in_path("fsl*-bet") if bet: - config['prefix'] = os.path.basename(bet)[:-3] + config["prefix"] = os.path.basename(bet)[:-3] def activate_configurations(): - ''' + """ Activate the FSL module (set env variables) from the global configurations, in order to use them via :mod:`capsul.in_context.fsl` functions - ''' - conf = engine.configurations.get('capsul.engine.module.fsl', {}) - fsl_dir = conf.get('directory') + """ + conf = engine.configurations.get("capsul.engine.module.fsl", {}) + fsl_dir = conf.get("directory") if fsl_dir: - os.environ['FSLDIR'] = six.ensure_str(fsl_dir) - elif 'FSLDIR' in os.environ: - del os.environ['FSLDIR'] - fsl_prefix = conf.get('prefix') + os.environ["FSLDIR"] = six.ensure_str(fsl_dir) + elif "FSLDIR" in os.environ: + del os.environ["FSLDIR"] + fsl_prefix = conf.get("prefix") if fsl_prefix: - os.environ['FSL_PREFIX'] = six.ensure_str(fsl_prefix) - elif 'FSL_PREFIX' in os.environ: - del os.environ['FSL_PREFIX'] - fsl_conf = conf.get('config') + os.environ["FSL_PREFIX"] = six.ensure_str(fsl_prefix) + elif "FSL_PREFIX" in os.environ: + del os.environ["FSL_PREFIX"] + fsl_conf = conf.get("config") if fsl_conf: - os.environ['FSL_CONFIG'] = six.ensure_str(fsl_conf) - elif 'FSL_CONFIG' in os.environ: - del os.environ['FSL_CONFIG'] + os.environ["FSL_CONFIG"] = six.ensure_str(fsl_conf) + elif "FSL_CONFIG" in os.environ: + del os.environ["FSL_CONFIG"] -def edition_widget(engine, environment, config_id='fsl'): - ''' Edition GUI for FSL config - see +def edition_widget(engine, environment, config_id="fsl"): + """Edition GUI for FSL config - see :class:`~capsul.qt_gui.widgets.settings_editor.SettingsEditor` - ''' + """ from soma.qt_gui.controller_widget import ScrollControllerWidget from soma.controller import Controller import types @@ -126,8 +133,8 @@ def validate_config(widget): controller = widget.controller_widget.controller with widget.engine.settings as session: conf = session.config(config_id, widget.environment) - values = {'config_id': config_id} - for k in ('directory', 'config', 'prefix'): + values = {"config_id": config_id} + for k in ("directory", "config", "prefix"): value = getattr(controller, k) if value is traits.Undefined: value = None @@ -136,28 +143,32 @@ def validate_config(widget): session.new_config(config_id, widget.environment, values) else: for k, value in values.items(): - if k == 'config_id': + if k == "config_id": continue setattr(conf, k, values[k]) controller = Controller() - controller.add_trait('directory', traits.Directory(traits.Undefined, - desc='Directory where FSL is installed')) - controller.add_trait('config', traits.File( - traits.Undefined, - output=False, - desc='Parameter to specify the fsl.sh path')) - controller.add_trait('prefix', traits.String(traits.Undefined, - desc='Prefix to add to FSL commands')) - - conf = engine.settings.select_configurations( - environment, {'fsl': 'any'}) + controller.add_trait( + "directory", + traits.Directory(traits.Undefined, desc="Directory where FSL is installed"), + ) + controller.add_trait( + "config", + traits.File( + traits.Undefined, output=False, desc="Parameter to specify the fsl.sh path" + ), + ) + controller.add_trait( + "prefix", traits.String(traits.Undefined, desc="Prefix to add to FSL commands") + ) + + conf = engine.settings.select_configurations(environment, {"fsl": "any"}) if conf: - fconf = conf.get('capsul.engine.module.fsl', {}) - controller.directory = fconf.get('directory', traits.Undefined) - controller.config = fconf.get('config', traits.Undefined) - controller.prefix = fconf.get('prefix', traits.Undefined) + fconf = conf.get("capsul.engine.module.fsl", {}) + controller.directory = fconf.get("directory", traits.Undefined) + controller.config = fconf.get("config", traits.Undefined) + controller.prefix = fconf.get("prefix", traits.Undefined) widget = ScrollControllerWidget(controller, live=True) widget.engine = engine diff --git a/capsul/engine/module/matlab.py b/capsul/engine/module/matlab.py index 5347c4ee8..bcb18b12d 100644 --- a/capsul/engine/module/matlab.py +++ b/capsul/engine/module/matlab.py @@ -4,52 +4,63 @@ import capsul.engine import os -#from soma.controller import Controller -#from traits.api import File, Undefined, Instance +# from soma.controller import Controller +# from traits.api import File, Undefined, Instance + - def init_settings(capsul_engine): with capsul_engine.settings as settings: - settings.ensure_module_fields('matlab', - [dict(name='executable', - type='string', - description='Full path of the matlab executable'), - dict(name='mcr_directory', - type='string', - description='Full path of the matlab MCR directory, for use ' - 'with standalone binaries'), - ]) + settings.ensure_module_fields( + "matlab", + [ + dict( + name="executable", + type="str", + description="Full path of the matlab executable", + ), + dict( + name="mcr_directory", + type="str", + description="Full path of the matlab MCR directory, for use " + "with standalone binaries", + ), + ], + ) def check_configurations(): - ''' + """ Check if the activated configuration is valid for Matlab and return an error message if there is an error or None if everything is good. - ''' + """ matlab_executable = capsul.engine.configurations.get( - 'capsul.engine.module.matlab', {}).get('executable') - mcr = capsul.engine.configurations.get( - 'capsul.engine.module.matlab', {}).get('mcr_directory') + "capsul.engine.module.matlab", {} + ).get("executable") + mcr = capsul.engine.configurations.get("capsul.engine.module.matlab", {}).get( + "mcr_directory" + ) if not matlab_executable and not mcr: - return \ - 'both matlab.executable and matlab.mcr_directory are not defined' + return "both matlab.executable and matlab.mcr_directory are not defined" if matlab_executable and not os.path.exists(matlab_executable): - return 'Matlab executable is defined as "%s" but this path does not exist' % matlab_executable + return ( + 'Matlab executable is defined as "%s" but this path does not exist' + % matlab_executable + ) return None def check_notably_invalid_config(conf): - ''' + """ Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled Returns ------- invalid: list list of invalid config keys - ''' + """ invalid = [] - for k in ('executable', 'mcr_directory'): - if getattr(conf, k, None) is None: + for k in ("executable", "mcr_directory"): + if conf.get(k) is None: invalid.append(k) if len(invalid) == 1: # if one of the paths is filled, then it should be OK. @@ -59,10 +70,10 @@ def check_notably_invalid_config(conf): return invalid -def edition_widget(engine, environment, config_id='matlab'): - ''' Edition GUI for matlab config - see +def edition_widget(engine, environment, config_id="matlab"): + """Edition GUI for matlab config - see :class:`~capsul.qt_gui.widgets.settings_editor.SettingsEditor` - ''' + """ from soma.qt_gui.controller_widget import ScrollControllerWidget from soma.controller import Controller import types @@ -73,56 +84,62 @@ def validate_config(widget): controller = widget.controller_widget.controller with widget.engine.settings as session: conf = session.config(config_id, widget.environment) - values = {'config_id': config_id} - if controller.executable in (None, traits.Undefined, ''): - values['executable'] = None + values = {"config_id": config_id} + if controller.executable in (None, traits.Undefined, ""): + values["executable"] = None else: - values['executable'] = controller.executable - if controller.mcr_directory in (None, traits.Undefined, ''): - values['mcr_directory'] = None + values["executable"] = controller.executable + if controller.mcr_directory in (None, traits.Undefined, ""): + values["mcr_directory"] = None else: - values['mcr_directory'] = controller.mcr_directory + values["mcr_directory"] = controller.mcr_directory if conf is None: session.new_config(config_id, widget.environment, values) else: - for k in ('executable', 'mcr_directory', ): + for k in ( + "executable", + "mcr_directory", + ): # don't check invalid files: they may be valid on a remote # server - #if (k == 'mcr_directory' and - #values[k] and - #not os.path.isdir(values[k])): - #raise NotADirectoryError('\nMatlab mcr_directory ' - #'was not updated:\n{} is ' - #'not existing!'.format( - #values[k])) - #elif (k == 'executable' and - #values[k] and - #not os.path.isfile(values[k])): - #raise FileNotFoundError('\nMatlab executable ' - #'was not updated:\n{} is ' - #'not existing!'.format( - #values[k])) - #else: + # if (k == 'mcr_directory' and + # values[k] and + # not os.path.isdir(values[k])): + # raise NotADirectoryError('\nMatlab mcr_directory ' + #'was not updated:\n{} is ' + #'not existing!'.format( + # values[k])) + # elif (k == 'executable' and + # values[k] and + # not os.path.isfile(values[k])): + # raise FileNotFoundError('\nMatlab executable ' + #'was not updated:\n{} is ' + #'not existing!'.format( + # values[k])) + # else: setattr(conf, k, values[k]) controller = Controller() - controller.add_trait('executable', - traits.File(desc= - 'Full path of the matlab executable')) - controller.add_trait('mcr_directory', - traits.Directory(desc='Full path of the matlab MCR ' - 'directory, or use with standalone ' - 'binaries')) - - conf = engine.settings.select_configurations( - environment, {'matlab': 'any'}) + controller.add_trait( + "executable", traits.File(desc="Full path of the matlab executable") + ) + controller.add_trait( + "mcr_directory", + traits.Directory( + desc="Full path of the matlab MCR " + "directory, or use with standalone " + "binaries" + ), + ) + + conf = engine.settings.select_configurations(environment, {"matlab": "any"}) if conf: - controller.executable = conf.get( - 'capsul.engine.module.matlab', {}).get('executable', - traits.Undefined) - controller.mcr_directory = conf.get( - 'capsul.engine.module.matlab', {}).get('mcr_directory', - traits.Undefined) + controller.executable = conf.get("capsul.engine.module.matlab", {}).get( + "executable", traits.Undefined + ) + controller.mcr_directory = conf.get("capsul.engine.module.matlab", {}).get( + "mcr_directory", traits.Undefined + ) widget = ScrollControllerWidget(controller, live=True) widget.engine = engine diff --git a/capsul/engine/module/mrtrix.py b/capsul/engine/module/mrtrix.py index 4a5387c36..b35d7277f 100644 --- a/capsul/engine/module/mrtrix.py +++ b/capsul/engine/module/mrtrix.py @@ -8,22 +8,27 @@ def init_settings(capsul_engine): with capsul_engine.settings as settings: - settings.ensure_module_fields('mrtrix', - [dict(name='directory', - type='string', - description='Directory where mrtrix is installed') - ]) + settings.ensure_module_fields( + "mrtrix", + [ + dict( + name="directory", + type="str", + description="Directory where mrtrix is installed", + ) + ], + ) # init a single config - config = settings.config('mrtrix', 'global') + config = settings.config("mrtrix", "global") if not config: - settings.new_config('mrtrix', 'global', - {capsul_engine.settings.config_id_field: - 'mrtrix'}) + settings.new_config( + "mrtrix", "global", {capsul_engine.settings.config_id_field: "mrtrix"} + ) def check_notably_invalid_config(conf): - ''' + """ Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled @@ -31,32 +36,32 @@ def check_notably_invalid_config(conf): ------- invalid: list list of invalid config keys - ''' + """ invalid = [] - for k in ('directory', ): - if getattr(conf, k, None) is None: + for k in ("directory",): + if conf.get(k) is None: invalid.append(k) return invalid def activate_configurations(): - ''' + """ Activate the mrtrix module (set env variables) from the global configurations, in order to use them via :mod:`capsul.in_context.mrtrix` functions - ''' - conf = engine.configurations.get('capsul.engine.module.mrtrix', {}) - mrtrix_dir = conf.get('directory') + """ + conf = engine.configurations.get("capsul.engine.module.mrtrix", {}) + mrtrix_dir = conf.get("directory") if mrtrix_dir: - os.environ['MRTRIXPATH'] = six.ensure_str(mrtrix_dir) - elif 'MRTRIXPATH' in os.environ: - del os.environ['MRTRIXPATH'] + os.environ["MRTRIXPATH"] = six.ensure_str(mrtrix_dir) + elif "MRTRIXPATH" in os.environ: + del os.environ["MRTRIXPATH"] -def edition_widget(engine, environment, config_id='mrtrix'): - ''' Edition GUI for mrtrix config - see +def edition_widget(engine, environment, config_id="mrtrix"): + """Edition GUI for mrtrix config - see :class:`~capsul.qt_gui.widgets.settings_editor.SettingsEditor` - ''' + """ from soma.qt_gui.controller_widget import ScrollControllerWidget from soma.controller import Controller import types @@ -67,8 +72,8 @@ def validate_config(widget): controller = widget.controller_widget.controller with widget.engine.settings as session: conf = session.config(config_id, widget.environment) - values = {'config_id': config_id} - for k in ['directory']: + values = {"config_id": config_id} + for k in ["directory"]: value = getattr(controller, k) if value is traits.Undefined: value = None @@ -77,21 +82,21 @@ def validate_config(widget): session.new_config(config_id, widget.environment, values) else: for k, value in values.items(): - if k == 'config_id': + if k == "config_id": continue setattr(conf, k, values[k]) controller = Controller() - controller.add_trait('directory', - traits.Directory(traits.Undefined, - desc='Directory where mrtrix is installed')) + controller.add_trait( + "directory", + traits.Directory(traits.Undefined, desc="Directory where mrtrix is installed"), + ) - conf = engine.settings.select_configurations( - environment, {'mrtrix': 'any'}) + conf = engine.settings.select_configurations(environment, {"mrtrix": "any"}) if conf: - fconf = conf.get('capsul.engine.module.mrtrix', {}) - controller.directory = fconf.get('directory', traits.Undefined) + fconf = conf.get("capsul.engine.module.mrtrix", {}) + controller.directory = fconf.get("directory", traits.Undefined) widget = ScrollControllerWidget(controller, live=True) widget.engine = engine diff --git a/capsul/engine/module/python.py b/capsul/engine/module/python.py index 5f955546a..38454c974 100644 --- a/capsul/engine/module/python.py +++ b/capsul/engine/module/python.py @@ -27,10 +27,10 @@ def init_settings(capsul_engine): with capsul_engine.settings as settings: settings.ensure_module_fields('python', [dict(name='executable', - type='string', + type='str', description='Full path of the python executable'), dict(name='path', - type='list_string', + type='list[str]', description='paths to prepend to sys.path'), ]) @@ -56,7 +56,7 @@ def check_notably_invalid_config(conf): ''' invalid = [] for k in ('executable', ): - if getattr(conf, k, None) is None: + if conf.get(k) is None: invalid.append(k) return invalid diff --git a/capsul/engine/module/somaworkflow.py b/capsul/engine/module/somaworkflow.py index b4f0b1206..97fb7e1a9 100644 --- a/capsul/engine/module/somaworkflow.py +++ b/capsul/engine/module/somaworkflow.py @@ -7,56 +7,74 @@ def init_settings(capsul_engine): with capsul_engine.settings as settings: - settings.ensure_module_fields('somaworkflow', - [dict(name='use', - type='boolean', - description='Use soma workflow for the execution (not always ' - 'optional, actually)'), - dict(name='computing_resource', - type='string', - description='Soma-workflow computing resource to be used to ' - 'run processing'), - dict(name='config_file', - type='string', - description='Soma-Workflow configuration file. ' - 'Default: $HOME/.soma_workflow.cfg'), - dict(name='keep_failed_workflows', - type='boolean', - description='Keep failed workflows after pipeline execution ' - 'through StudyConfig'), - dict(name='keep_succeeded_workflows', - type='boolean', - description='Keep succeeded workflows after pipeline ' - 'execution through StudyConfig'), - - dict(name='queue', - type='string', - description='Jobs queue to be used on the computing resource ' - 'for workflow submissions'), - dict(name='transfer_paths', - type='list_string', - description='list of paths where files have to be transferred ' - 'by soma-workflow'), - dict(name='path_translations', - type='json', - description='Soma-workflow paths translations mapping: ' - '{local_path: (identifier, uuid)}'), - ]) + settings.ensure_module_fields( + "somaworkflow", + [ + dict( + name="use", + type="bool", + description="Use soma workflow for the execution (not always " + "optional, actually)", + ), + dict( + name="computing_resource", + type="str", + description="Soma-workflow computing resource to be used to " + "run processing", + ), + dict( + name="config_file", + type="str", + description="Soma-Workflow configuration file. " + "Default: $HOME/.soma_workflow.cfg", + ), + dict( + name="keep_failed_workflows", + type="bool", + description="Keep failed workflows after pipeline execution " + "through StudyConfig", + ), + dict( + name="keep_succeeded_workflows", + type="bool", + description="Keep succeeded workflows after pipeline " + "execution through StudyConfig", + ), + dict( + name="queue", + type="str", + description="Jobs queue to be used on the computing resource " + "for workflow submissions", + ), + dict( + name="transfer_paths", + type="list[str]", + description="list of paths where files have to be transferred " + "by soma-workflow", + ), + dict( + name="path_translations", + type="dict", + description="Soma-workflow paths translations mapping: " + "{local_path: (identifier, uuid)}", + ), + ], + ) initialize_callbacks(capsul_engine) def activate_configurations(): - ''' + """ Activate the SPM module (set env variables) from the global configurations, in order to use them via :mod:`capsul.in_context.spm` functions - ''' - conf = engine.configurations.get('capsul.engine.module.somaworkflow', {}) + """ + conf = engine.configurations.get("capsul.engine.module.somaworkflow", {}) -def edition_widget(engine, environment, config_id='any'): - ''' Edition GUI for SPM config - see +def edition_widget(engine, environment, config_id="any"): + """Edition GUI for SPM config - see :class:`~capsul.qt_gui.widgets.settings_editor.SettingsEditor` - ''' + """ from soma.qt_gui.controller_widget import ScrollControllerWidget from soma.controller import Controller import types @@ -66,61 +84,72 @@ def validate_config(widget): controller = widget.controller_widget.controller with widget.engine.settings as session: values = {} - for key in ('computing_resource', 'config_file', 'queue'): - if getattr(controller, key) in (None, traits.Undefined, ''): + for key in ("computing_resource", "config_file", "queue"): + if getattr(controller, key) in (None, traits.Undefined, ""): values[key] = None else: values[key] = getattr(controller, key) - for key in ('use', 'keep_failed_workflows', - 'keep_succeeded_workflows', 'transfer_paths', - #'path_translations', - ): + for key in ( + "use", + "keep_failed_workflows", + "keep_succeeded_workflows", + "transfer_paths", + #'path_translations', + ): values[key] = getattr(controller, key) - id = 'somaworkflow' - values['config_id'] = id + id = "somaworkflow" + values["config_id"] = id query = 'config_id == "%s"' % id - conf = session.config('somaworkflow', widget.environment, - selection=query) + conf = session.config("somaworkflow", widget.environment, selection=query) if conf is None: - session.new_config('somaworkflow', widget.environment, values) + session.new_config("somaworkflow", widget.environment, values) else: - for k in ('computing_resource', 'config_file', 'queue', 'use', - 'keep_failed_workflows', 'keep_succeeded_workflows', - 'transfer_paths', - #'path_translations' - ): + for k in ( + "computing_resource", + "config_file", + "queue", + "use", + "keep_failed_workflows", + "keep_succeeded_workflows", + "transfer_paths", + #'path_translations' + ): setattr(conf, k, values[k]) if id != widget.config_id: try: - session.remove_config('somaworkflow', widget.environment, - widget.config_id) + session.remove_config( + "somaworkflow", widget.environment, widget.config_id + ) except Exception: pass widget.config_id = id controller = Controller() - controller.add_trait('use', - traits.Bool( - True, output=False, - desc='Use soma workflow for the execution (not always optional, actually)')) - controller.add_trait('computing_resource', - traits.Str( - traits.Undefined, output=False, - desc='Soma-workflow computing resource to be used ' - 'to run processing')) - controller.add_trait('config_file', - traits.File( - traits.Undefined, output=False, - desc='')) - controller.add_trait('keep_failed_workflows', - traits.Bool( - True, output=False, - desc='')) - controller.add_trait('keep_succeeded_workflows', - traits.Bool( - False, output=False, - desc='')) - + controller.add_trait( + "use", + traits.Bool( + True, + output=False, + desc="Use soma workflow for the execution (not always optional, actually)", + ), + ) + controller.add_trait( + "computing_resource", + traits.Str( + traits.Undefined, + output=False, + desc="Soma-workflow computing resource to be used " "to run processing", + ), + ) + controller.add_trait( + "config_file", traits.File(traits.Undefined, output=False, desc="") + ) + controller.add_trait( + "keep_failed_workflows", traits.Bool(True, output=False, desc="") + ) + controller.add_trait( + "keep_succeeded_workflows", traits.Bool(False, output=False, desc="") + ) controller.add_trait('queue', traits.Str( traits.Undefined, output=False, @@ -136,41 +165,46 @@ def validate_config(widget): maxlen=2), value={}, output=False, desc='')) - conf = None - if config_id == 'any': + if config_id == "any": conf = engine.settings.select_configurations( - environment, {'somaworkflow': 'any'}) + environment, {"somaworkflow": "any"} + ) else: try: conf = engine.settings.select_configurations( - environment, {'somaworkflow': 'config_id=="%s"' % config_id}) + environment, {"somaworkflow": 'config_id=="%s"' % config_id} + ) except Exception: pass if conf: - controller.use = conf.get( - 'capsul.engine.module.somaworkflow', {}).get('use', True) + controller.use = conf.get("capsul.engine.module.somaworkflow", {}).get( + "use", True + ) controller.computing_resource = conf.get( - 'capsul.engine.module.somaworkflow', {}).get('computing_resource', - traits.Undefined) - controller.config_file = conf.get( - 'capsul.engine.module.somaworkflow', {}).get('config_file', - traits.Undefined) - config_id = conf.get( - 'capsul.engine.module.somaworkflow', {}).get('config_id', config_id) + "capsul.engine.module.somaworkflow", {} + ).get("computing_resource", traits.Undefined) + controller.config_file = conf.get("capsul.engine.module.somaworkflow", {}).get( + "config_file", traits.Undefined + ) + config_id = conf.get("capsul.engine.module.somaworkflow", {}).get( + "config_id", config_id + ) controller.keep_failed_workflows = conf.get( - 'capsul.engine.module.somaworkflow', - {}).get('keep_failed_workflows', True) + "capsul.engine.module.somaworkflow", {} + ).get("keep_failed_workflows", True) controller.keep_succeeded_workflows = conf.get( - 'capsul.engine.module.somaworkflow', - {}).get('keep_succeeded_workflows', False) - controller.queue = conf.get( - 'capsul.engine.module.somaworkflow', {}).get('queue', Undefined) + "capsul.engine.module.somaworkflow", {} + ).get("keep_succeeded_workflows", False) + controller.queue = conf.get("capsul.engine.module.somaworkflow", {}).get( + "queue", Undefined + ) controller.transfer_paths = conf.get( - 'capsul.engine.module.somaworkflow', {}).get('transfer_paths', []) + "capsul.engine.module.somaworkflow", {} + ).get("transfer_paths", []) controller.path_translations = conf.get( - 'capsul.engine.module.somaworkflow', {}).get( - 'path_translations', {}) + "capsul.engine.module.somaworkflow", {} + ).get("path_translations", {}) # TODO handle several configs @@ -187,38 +221,38 @@ def initialize_callbacks(engine): # WARNING ref to engine in callback engine.study_config.on_trait_change( lambda param=None, value=None: sync_from_sc(engine, param, value), - '[use_soma_workflow, somaworkflow_computing_resource, ' - 'somaworkflow_config_file, somaworkflow_keep_failed_workflows, ' - 'somaworkflow_keep_succeeded_workflows, ' - 'somaworkflow_computing_resources_config]') + "[use_soma_workflow, somaworkflow_computing_resource, " + "somaworkflow_config_file, somaworkflow_keep_failed_workflows, " + "somaworkflow_keep_succeeded_workflows, " + "somaworkflow_computing_resources_config]", + ) # WARNING ref to engine in callback engine.study_config.engine.settings.module_notifiers[ - 'capsul.engine.module.somaworkflow'] \ - = [lambda param=None, value=None: sync_to_sc(engine, param, value)] + "capsul.engine.module.somaworkflow" + ] = [lambda param=None, value=None: sync_to_sc(engine, param, value)] def sync_from_sc(engine, param=None, value=None): # print('sync_from_sc') - if getattr(engine, '_swf_syncing', False): + if getattr(engine, "_swf_syncing", False): # manage recursive calls return engine._swf_syncing = True sc = engine.study_config try: - if 'SomaWorkflowConfig' in sc.modules \ - and 'capsul.engine.module.somaworkflow' \ - in engine._loaded_modules: + if ( + "SomaWorkflowConfig" in sc.modules + and "capsul.engine.module.somaworkflow" in engine._loaded_modules + ): with engine.settings as session: cif = engine.settings.config_id_field - config = session.config('somaworkflow', 'global') + config = session.config("somaworkflow", "global") params = { - 'use_soma_workflow': 'use', - 'somaworkflow_computing_resource': 'computing_resource', - 'somaworkflow_config_file': 'config_file', - 'somaworkflow_keep_failed_workflows': - 'keep_failed_workflows', - 'somaworkflow_keep_succeeded_workflows': - 'keep_succeeded_workflows', + "use_soma_workflow": "use", + "somaworkflow_computing_resource": "computing_resource", + "somaworkflow_config_file": "config_file", + "somaworkflow_keep_failed_workflows": "keep_failed_workflows", + "somaworkflow_keep_succeeded_workflows": "keep_succeeded_workflows", } values = {} for sc_param, param in params.items(): @@ -227,8 +261,8 @@ def sync_from_sc(engine, param=None, value=None): value = None values[param] = value if config is None: - values[cif] = 'somaworkflow' - session.new_config('somaworkflow', 'global', values) + values[cif] = "somaworkflow" + session.new_config("somaworkflow", "global", values) else: for param, value in values.items(): setattr(config, param, value) @@ -240,27 +274,25 @@ def sync_from_sc(engine, param=None, value=None): def sync_to_sc(engine, param=None, value=None): # print('sync_to_sc') - if getattr(engine, '_swf_syncing', False): + if getattr(engine, "_swf_syncing", False): # manage recursive calls return engine._swf_syncing = True sc = engine.study_config try: - if 'SomaWorkflowConfig' in sc.modules \ - and 'capsul.engine.module.somaworkflow' \ - in engine._loaded_modules: + if ( + "SomaWorkflowConfig" in sc.modules + and "capsul.engine.module.somaworkflow" in engine._loaded_modules + ): with engine.settings as session: - config = session.config('somaworkflow', 'global') + config = session.config("somaworkflow", "global") if config: params = { - 'use': 'use_soma_workflow', - 'computing_resource': - 'somaworkflow_computing_resource', - 'config_file': 'somaworkflow_config_file', - 'keep_failed_workflows': - 'somaworkflow_keep_failed_workflows', - 'keep_succeeded_workflows': - 'somaworkflow_keep_succeeded_workflows', + "use": "use_soma_workflow", + "computing_resource": "somaworkflow_computing_resource", + "config_file": "somaworkflow_config_file", + "keep_failed_workflows": "somaworkflow_keep_failed_workflows", + "keep_succeeded_workflows": "somaworkflow_keep_succeeded_workflows", } for param, sc_param in params.items(): value = getattr(config, param, Undefined) diff --git a/capsul/engine/module/spm.py b/capsul/engine/module/spm.py index 6827ff351..44483fd52 100644 --- a/capsul/engine/module/spm.py +++ b/capsul/engine/module/spm.py @@ -5,98 +5,108 @@ import os import six -#import glob -#import os.path as osp -#import weakref +# import glob +# import os.path as osp +# import weakref ##import subprocess # Only in case of matlab call (auto_configuration func) -#from soma.controller import Controller -#from traits.api import Directory, Undefined, Instance, String, Bool +# from soma.controller import Controller +# from traits.api import Directory, Undefined, Instance, String, Bool + +# from . import matlab -#from . import matlab - def init_settings(capsul_engine): with capsul_engine.settings as settings: - settings.ensure_module_fields('spm', - [dict(name='directory', - type='string', - description='Directory where SPM is installed'), - dict(name='version', - type='string', - description='Version of SPM release (8 or 12)'), - dict(name='standalone', - type='boolean', - description='If this parameter is set to True, use the ' - 'standalone SPM version, otherwise use Matlab.') - ]) + settings.ensure_module_fields( + "spm", + [ + dict( + name="directory", + type="str", + description="Directory where SPM is installed", + ), + dict( + name="version", + type="str", + description="Version of SPM release (8 or 12)", + ), + dict( + name="standalone", + type="bool", + description="If this parameter is set to True, use the " + "standalone SPM version, otherwise use Matlab.", + ), + ], + ) def config_dependencies(config): - return {'matlab': 'any'} + return {"matlab": "any"} + + +# def set_environ(config, environ): +# spm_config = config.get('spm', {}) +# use = spm_config.get('use') +# if use is True or (use is None and 'directory' in spm_config): +# error_message = check_environ(environ) +# if error_message: +# complete_environ(config, environ) +# error_message = check_environ(environ) + +# if error_message: +# raise EnvironmentError(error_message) + -#def set_environ(config, environ): - #spm_config = config.get('spm', {}) - #use = spm_config.get('use') - #if use is True or (use is None and 'directory' in spm_config): - #error_message = check_environ(environ) - #if error_message: - #complete_environ(config, environ) - #error_message = check_environ(environ) +# def check_environ(environ): +#''' +# Check if the configuration is valid to run SPM and returns an error +# message if there is an error or None if everything is good. +#''' +# if not environ.get('SPM_DIRECTORY'): +# return 'SPM directory is not defined' +# if not environ.get('SPM_VERSION'): +# return 'SPM version is not defined (maybe %s is not a valid SPM directory)' % environ['SPM_DIRECTORY'] +# if not environ.get('SPM_STANDALONE'): +# return 'No selection of SPM installation type : Standalone or Matlab' +# if not osp.isdir(environ['SPM_DIRECTORY']): +# return 'No valid SPM directory: %s' % environ['SPM_DIRECTORY'] +# if environ['SPM_STANDALONE'] != 'yes': +# matlab_error = matlab.check_environ(environ) +# if matlab_error: +# return 'Matlab configuration must be valid for SPM: ' + matlab_error +# return None - #if error_message: - #raise EnvironmentError(error_message) +# def complete_environ(config, environ): +#''' +# Try to automatically complete environment for SPM +#''' +# spm_directory = config.get('spm', {}).get('directory') +# if spm_directory: +# environ['SPM_DIRECTORY'] = spm_directory +# mcr = glob.glob(osp.join(spm_directory, 'spm*_mcr')) +# if mcr: +# fileName = osp.basename(mcr[0]) +# inc = 1 - -#def check_environ(environ): - #''' - #Check if the configuration is valid to run SPM and returns an error - #message if there is an error or None if everything is good. - #''' - #if not environ.get('SPM_DIRECTORY'): - #return 'SPM directory is not defined' - #if not environ.get('SPM_VERSION'): - #return 'SPM version is not defined (maybe %s is not a valid SPM directory)' % environ['SPM_DIRECTORY'] - #if not environ.get('SPM_STANDALONE'): - #return 'No selection of SPM installation type : Standalone or Matlab' - #if not osp.isdir(environ['SPM_DIRECTORY']): - #return 'No valid SPM directory: %s' % environ['SPM_DIRECTORY'] - #if environ['SPM_STANDALONE'] != 'yes': - #matlab_error = matlab.check_environ(environ) - #if matlab_error: - #return 'Matlab configuration must be valid for SPM: ' + matlab_error - #return None +# while fileName[fileName.find('spm') + 3: +# fileName.find('spm') + 3 + inc].isdigit(): +# environ['SPM_VERSION'] = fileName[fileName.find('spm') + 3: +# fileName.find('spm') + 3 +# + inc] +# inc+=1 -#def complete_environ(config, environ): - #''' - #Try to automatically complete environment for SPM - #''' - #spm_directory = config.get('spm', {}).get('directory') - #if spm_directory: - #environ['SPM_DIRECTORY'] = spm_directory - #mcr = glob.glob(osp.join(spm_directory, 'spm*_mcr')) - #if mcr: - #fileName = osp.basename(mcr[0]) - #inc = 1 +# environ['SPM_STANDALONE'] = 'yes' - #while fileName[fileName.find('spm') + 3: - #fileName.find('spm') + 3 + inc].isdigit(): - #environ['SPM_VERSION'] = fileName[fileName.find('spm') + 3: - #fileName.find('spm') + 3 - #+ inc] - #inc+=1 - - #environ['SPM_STANDALONE'] = 'yes' - - #else: - #environ['SPM_STANDALONE'] = 'no' - ## determine SPM version (currently 8 or 12) - #if osp.isdir(osp.join(spm_directory, 'toolbox', 'OldNorm')): - #environ['SPM_VERSION'] = '12' - #elif osp.isdir(osp.join(spm_directory, 'templates')): - #environ['SPM_VERSION'] = '8' - #else: - #environ.pop('SPM_VERSION', None) +# else: +# environ['SPM_STANDALONE'] = 'no' +## determine SPM version (currently 8 or 12) +# if osp.isdir(osp.join(spm_directory, 'toolbox', 'OldNorm')): +# environ['SPM_VERSION'] = '12' +# elif osp.isdir(osp.join(spm_directory, 'templates')): +# environ['SPM_VERSION'] = '8' +# else: +# environ.pop('SPM_VERSION', None) # For SPM with MATLAB license, if we want to get the SPM version from a system # call to matlab:. @@ -133,54 +143,51 @@ def config_dependencies(config): def activate_configurations(): - ''' + """ Activate the SPM module (set env variables) from the global configurations, in order to use them via :mod:`capsul.in_context.spm` functions - ''' - conf = engine.configurations.get('capsul.engine.module.spm', {}) - mlab_conf = engine.configurations.get('capsul.engine.module.matlab', {}) - spm_dir = conf.get('directory') - mcr_dir = mlab_conf.get('mcr_directory') + """ + conf = engine.configurations.get("capsul.engine.module.spm", {}) + mlab_conf = engine.configurations.get("capsul.engine.module.matlab", {}) + spm_dir = conf.get("directory") + mcr_dir = mlab_conf.get("mcr_directory") if spm_dir: - os.environ['SPM_DIRECTORY'] = six.ensure_str(spm_dir) - elif 'SPM_DIRECTORY' in os.environ: - del os.environ['SPM_DIRECTORY'] - spm_version = conf.get('version') + os.environ["SPM_DIRECTORY"] = six.ensure_str(spm_dir) + elif "SPM_DIRECTORY" in os.environ: + del os.environ["SPM_DIRECTORY"] + spm_version = conf.get("version") if spm_version: - os.environ['SPM_VERSION'] = six.ensure_str(spm_version) - elif 'SPM_VERSION' in os.environ: - del os.environ['SPM_VERSION'] - spm_standalone = conf.get('standalone') + os.environ["SPM_VERSION"] = six.ensure_str(spm_version) + elif "SPM_VERSION" in os.environ: + del os.environ["SPM_VERSION"] + spm_standalone = conf.get("standalone") if spm_standalone is not None: - os.environ['SPM_STANDALONE'] = 'yes' if spm_standalone else 'no' + os.environ["SPM_STANDALONE"] = "yes" if spm_standalone else "no" if spm_standalone and mcr_dir: - os.environ['MCR_HOME'] = mcr_dir - elif 'SPM_STANDALONE' in os.environ: - del os.environ['SPM_STANDALONE'] + os.environ["MCR_HOME"] = mcr_dir + elif "SPM_STANDALONE" in os.environ: + del os.environ["SPM_STANDALONE"] def check_notably_invalid_config(conf): - ''' + """ Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled Returns ------- invalid: list list of invalid config keys - ''' + """ invalid = [] - #for k in ('directory', 'version'): - # if getattr(conf, k, None) is None: - # invalid.append(k) - if getattr(conf, 'directory', None) is None: - invalid.append('directory') + if conf.get("directory") is None: + invalid.append("directory") return invalid -def edition_widget(engine, environment, config_id='any'): - ''' Edition GUI for SPM config - see +def edition_widget(engine, environment, config_id="any"): + """Edition GUI for SPM config - see :class:`~capsul.qt_gui.widgets.settings_editor.SettingsEditor` - ''' + """ from soma.qt_gui.controller_widget import ScrollControllerWidget from soma.controller import Controller import types @@ -191,73 +198,82 @@ def validate_config(widget): controller = widget.controller_widget.controller with widget.engine.settings as session: values = {} - if controller.directory in (None, traits.Undefined, ''): - values['directory'] = None + if controller.directory in (None, traits.Undefined, ""): + values["directory"] = None else: - values['directory'] = controller.directory - values['standalone'] = controller.standalone - if controller.version in (None, traits.Undefined, ''): - values['version'] = None + values["directory"] = controller.directory + values["standalone"] = controller.standalone + if controller.version in (None, traits.Undefined, ""): + values["version"] = None else: - values['version'] = controller.version - id = 'spm%s%s' % (controller.version if - controller.version != traits.Undefined else '', - '-standalone' if controller.standalone else '') - values['config_id'] = id + values["version"] = controller.version + id = "spm%s%s" % ( + controller.version if controller.version != traits.Undefined else "", + "-standalone" if controller.standalone else "", + ) + values["config_id"] = id query = 'config_id == "%s"' % id - conf = session.config('spm', widget.environment, selection=query) + conf = session.config("spm", widget.environment, selection=query) if conf is None: - session.new_config('spm', widget.environment, values) + session.new_config("spm", widget.environment, values) else: - for k in ('directory', 'standalone', 'version'): - if (k == 'directory' and - values[k] and - not os.path.isdir(values[k])): - raise NotADirectoryError('\nSPM directory was not ' - 'updated:\n{} is not ' - 'existing!'.format(values[k])) + for k in ("directory", "standalone", "version"): + if k == "directory" and values[k] and not os.path.isdir(values[k]): + raise NotADirectoryError( + "\nSPM directory was not " + "updated:\n{} is not " + "existing!".format(values[k]) + ) else: setattr(conf, k, values[k]) if id != widget.config_id: try: - session.remove_config('spm', widget.environment, - widget.config_id) + session.remove_config("spm", widget.environment, widget.config_id) except Exception: pass widget.config_id = id controller = Controller() - controller.add_trait("directory", traits.Directory( - traits.Undefined, - output=False, - desc="Directory containing SPM.")) - controller.add_trait("standalone", traits.Bool( - False, - desc="If True, use the standalone version of SPM.")) - controller.add_trait('version', traits.Str( - traits.Undefined, output=False, - desc='Version string for SPM: "12", "8", etc.')) + controller.add_trait( + "directory", + traits.Directory( + traits.Undefined, output=False, desc="Directory containing SPM." + ), + ) + controller.add_trait( + "standalone", + traits.Bool(False, desc="If True, use the standalone version of SPM."), + ) + controller.add_trait( + "version", + traits.Str( + traits.Undefined, + output=False, + desc='Version string for SPM: "12", "8", etc.', + ), + ) conf = None - if config_id == 'any': - conf = engine.settings.select_configurations( - environment, {'spm': 'any'}) + if config_id == "any": + conf = engine.settings.select_configurations(environment, {"spm": "any"}) else: try: conf = engine.settings.select_configurations( - environment, {'spm': 'config_id=="%s"' % config_id}) + environment, {"spm": 'config_id=="%s"' % config_id} + ) except Exception: pass if conf: - controller.directory = conf.get( - 'capsul.engine.module.spm', {}).get('directory', - traits.Undefined) - controller.standalone = conf.get( - 'capsul.engine.module.spm', {}).get('standalone', False) - controller.version = conf.get( - 'capsul.engine.module.spm', {}).get('version', traits.Undefined) - config_id = conf.get( - 'capsul.engine.module.spm', {}).get('config_id', config_id) + controller.directory = conf.get("capsul.engine.module.spm", {}).get( + "directory", traits.Undefined + ) + controller.standalone = conf.get("capsul.engine.module.spm", {}).get( + "standalone", False + ) + controller.version = conf.get("capsul.engine.module.spm", {}).get( + "version", traits.Undefined + ) + config_id = conf.get("capsul.engine.module.spm", {}).get("config_id", config_id) # TODO handle several configs diff --git a/capsul/engine/settings.py b/capsul/engine/settings.py index 9fade0219..4ceb7b9e4 100644 --- a/capsul/engine/settings.py +++ b/capsul/engine/settings.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -''' +""" This module provides classes to store CapsulEngine settings for several execution environment and choose a configuration for a given execution environment. Setting management in Capsul has several features that makes it different from classical ways to deal with configuration: * CapsulEngine must be able to deal with several configurations for the same software. For instance, one can configure both SPM 8 and SPM 12 and choose later the one to use. @@ -16,7 +16,7 @@ Settings cannot be used directly to configure the execution of a software. It is necessary to first select a single configuration document for each module. This configurations selection step is done by the :meth:`Settings.select_configurations` method. -''' +""" # # Questions about settings @@ -44,7 +44,7 @@ class Settings: - ''' + """ Main class for the management of CapsulEngine settings. Since these settings are always stored in a populse_db database, it is necessary to activate a settings session in order to read or modify settings. This is @@ -60,64 +60,63 @@ class Settings: {'version': '12', 'standalone': True}) # modify value conf.directory = '/usr/local/spm12-standalone' - ''' + """ - global_environment = 'global' - collection_prefix = 'settings/' - environment_field = 'config_environment' - config_id_field = 'config_id' + global_environment = "global" + collection_prefix = "settings/" + environment_field = "config_environment" + config_id_field = "config_id" def __init__(self, populse_db): - ''' + """ Create a settings instance using the given populse_db instance - ''' + """ self.populse_db = populse_db self.module_notifiers = {} def __enter__(self): - ''' + """ Starts a session to read or write settings - ''' - dbs = self.populse_db.__enter__() - return SettingsSession(dbs, module_notifiers=self.module_notifiers) + """ + + return SettingsSession(self.populse_db, module_notifiers=self.module_notifiers) def __exit__(self, *args): - self.populse_db.__exit__(*args) + pass @staticmethod def module_name(module_name): - ''' + """ Return a complete module name (which must be a valid Python module name) given a possibly abbreviated module name. This method must be used whenever a module name is written by a user (for instance in a configuration file. This method add the prefix `'capsul.engine.module.'` if the module name does not contain a dot. - ''' - if '.' not in module_name: - module_name = 'capsul.engine.module.' + module_name + """ + if "." not in module_name: + module_name = "capsul.engine.module." + module_name return module_name - - def select_configurations(self, environment, uses=None, - check_invalid_mods=False): - ''' + + def select_configurations(self, environment, uses=None, check_invalid_mods=False): + """ Select a configuration for a given environment. A configuration is a dictionary whose keys are module names and values are configuration documents. The returned set of configuration per module can be activaded with `capsul.api.activate_configuration()`. - + The `uses` parameter determine which modules - must be included in the configuration. If not given, this method + must be included in the configuration. If not given, this method considers all configurations for every module defined in settings. This parameter is a dictionary whose keys are a module name and values are populse_db queries used to select module. - + The environment parameter defines the execution environment in which the configurations will be used. For each module, configurations are filtered with the query. First, values are searched in the given environment and, if no result is found, the `'global'` environment (the value defined in `Settings.global_environment`) is used. - + If `check_invalid_mods` is True, then each selected config module is checked for missing values and discarded if there are. @@ -128,44 +127,44 @@ def select_configurations(self, environment, uses=None, config = ce.select_configurations('my_environment', uses={'spm': 'version > 8'}) - ''' + """ configurations = {} with self as settings: if uses is None: uses = {} - for collection in (i.collection_name - for i in - settings._dbs.get_collections()): - if collection.startswith(Settings.collection_prefix): - module_name = \ - collection[len(Settings.collection_prefix):] - uses[module_name] = 'ALL' + with settings._storage.data() as data: + for collection in data.collection_names(): + if collection.startswith(Settings.collection_prefix): + module_name = collection[len(Settings.collection_prefix) :] + uses[module_name] = "ALL" uses_stack = list(uses.items()) while uses_stack: module, query = uses_stack.pop(-1) # append environment to config_id if it is given in the filter if 'config_id=="' in query: i = query.index('config_id=="') - i = query.index('"', i+12) - query = '%s-%s%s' % (query[:i], environment, query[i:]) + i = query.index('"', i + 12) + query = "%s-%s%s" % (query[:i], environment, query[i:]) module = self.module_name(module) if module in configurations: continue - configurations.setdefault('capsul_engine', - {}).setdefault('uses', - {})[module] = query + configurations.setdefault("capsul_engine", {}).setdefault("uses", {})[ + module + ] = query selected_config = None query_env = environment full_query = '%s == "%s" AND (%s)' % ( - Settings.environment_field, environment, ( - 'ALL' if query == 'any' else query)) - collection = '%s%s' % (Settings.collection_prefix, module) - if settings._dbs.get_collection(collection): - docs = list(settings._dbs.filter_documents(collection, - full_query)) - else: - docs = [] + Settings.environment_field, + environment, + ("ALL" if query == "any" else query), + ) + collection = "%s%s" % (Settings.collection_prefix, module) + with settings._storage.data() as data: + if data.has_collection(collection): + docs = data[collection].search(full_query) + else: + docs = [] if check_invalid_mods: # filter out invalid configs @@ -174,7 +173,7 @@ def select_configurations(self, environment, uses=None, for doc in doc_t: mod = sys.modules[module] invalid = [] - if hasattr(mod, 'check_notably_invalid_config'): + if hasattr(mod, "check_notably_invalid_config"): invalid = mod.check_notably_invalid_config(doc) if len(invalid) == 0: docs.append(doc) @@ -182,24 +181,27 @@ def select_configurations(self, environment, uses=None, if len(docs) == 1: selected_config = docs[0] elif len(docs) > 1: - if query == 'any': + if query == "any": selected_config = docs[0] else: - raise EnvironmentError('Cannot create configurations ' + raise EnvironmentError( + "Cannot create configurations " 'for environment "%s" because settings returned ' - '%d instances for module %s' % (environment, - len(docs), module)) + "%d instances for module %s" + % (environment, len(docs), module) + ) else: - full_query = '%s == "%s" AND (%s)' % (Settings.environment_field, - Settings.global_environment, - ('ALL' if query == 'any' - else query)) + full_query = '%s == "%s" AND (%s)' % ( + Settings.environment_field, + Settings.global_environment, + ("ALL" if query == "any" else query), + ) query_env = Settings.global_environment - if settings._dbs.get_collection(collection): - docs = list(settings._dbs.filter_documents(collection, - full_query)) - else: - docs = [] + with settings._storage.data() as data: + if data.has_collection(collection): + docs = data[collection].search(full_query) + else: + docs = [] if check_invalid_mods: # filter out invalid configs @@ -208,7 +210,7 @@ def select_configurations(self, environment, uses=None, for doc in doc_t: mod = sys.modules[module] invalid = [] - if hasattr(mod, 'check_notably_invalid_config'): + if hasattr(mod, "check_notably_invalid_config"): invalid = mod.check_notably_invalid_config(doc) if len(invalid) == 0: docs.append(doc) @@ -216,38 +218,40 @@ def select_configurations(self, environment, uses=None, if len(docs) == 1: selected_config = docs[0] elif len(docs) > 1: - if query == 'any': + if query == "any": selected_config = docs[0] else: - raise EnvironmentError('Cannot create ' + raise EnvironmentError( + "Cannot create " 'configurations for environment "%s" because ' - 'global settings returned %d instances for ' - 'module %s' % (query_env, len(docs), module)) + "global settings returned %d instances for " + "module %s" % (query_env, len(docs), module) + ) if selected_config: # Remove values that are None - items = getattr(selected_config, '_items', None) + items = getattr(selected_config, "_items", None) if items is None: # older populse_db 1.x items = selected_config.items selected_config = dict(items()) - if 'config_id' in selected_config: - selected_config['config_id'] \ - = selected_config['config_id'][ - :-len(query_env)-1] + if "config_id" in selected_config: + selected_config["config_id"] = selected_config["config_id"][ + : -len(query_env) - 1 + ] for k, v in list(items()): if v is None: del selected_config[k] configurations[module] = selected_config python_module = importlib.import_module(module) - config_dependencies = getattr(python_module, - 'config_dependencies', - None) + config_dependencies = getattr( + python_module, "config_dependencies", None + ) if config_dependencies: d = config_dependencies(selected_config) if d: uses_stack.extend( - [(Settings.module_name(k), v) - for k, v in d.items()]) + [(Settings.module_name(k), v) for k, v in d.items()] + ) return configurations @@ -259,95 +263,97 @@ def export_config_dict(self, environment=None): environment = [environment] with self as session: modules = [] - for collection in (i.collection_name - for i in - session._dbs.get_collections()): - if collection.startswith(Settings.collection_prefix): - module_name = \ - collection[len(Settings.collection_prefix):] - modules.append(module_name) - - for env in environment: - env_conf = {} - for module in modules: - mod_conf = {} - for config in session.configs(module, env): - id = '%s-%s' % (config._id, env) - doc = session._dbs.get_document( - session.collection_name(module), id) - items = dict(doc._items()) - if 'config_id' in items: - items['config_id'] \ - = items['config_id'][:-len(env)-1] - mod_conf[config._id] = items - if mod_conf: - env_conf[module] = mod_conf - - env_conf['capsul_engine'] = {'uses': {m: 'ALL' - for m in modules}} - - conf[env] = env_conf + with session._storage.data() as data: + for collection in data.collection_names(): + if collection.startswith(Settings.collection_prefix): + module_name = collection[len(Settings.collection_prefix) :] + modules.append(module_name) + + for env in environment: + env_conf = {} + for module in modules: + mod_conf = {} + for config in session.configs(module, env): + id = f"{config._id}-{env}" + doc = data[session.collection_name(module)][id].get() + if "config_id" in doc: + doc["config_id"] = doc["config_id"][: -len(env) - 1] + mod_conf[config._id] = doc + if mod_conf: + env_conf[module] = mod_conf + + env_conf["capsul_engine"] = {"uses": {m: "ALL" for m in modules}} + + conf[env] = env_conf return conf - def import_configs(self, environment, config_dict, cont_on_error=False): - ''' + """ Import config values from a dictionary as given by :meth:`select_configurations`. Compared to :meth:`CapsulEngine.import_configs` this method (at :class:`Settings` level) does not load the required modules. - ''' - modules = config_dict.get('capsul_engine', {}).get('uses', {}) + """ + modules = config_dict.get("capsul_engine", {}).get("uses", {}) with self as session: for module in modules: mod_dict = config_dict.get(module, {}) if mod_dict: - if 'config_id' in mod_dict: + if "config_id" in mod_dict: # select_configurations() shape: 1 config per module - mod_dict = {mod_dict['config_id']: mod_dict} + mod_dict = {mod_dict["config_id"]: mod_dict} # otherwise several config are indexed by their config_id for config_id, config in mod_dict.items(): conf = session.config( - module, environment, 'config_id == "%s"' % - config_id) + module, environment, 'config_id == "%s"' % config_id + ) try: if conf: - values = {k: v for k, v in config.items() - if k not in ('config_id', - 'config_environment')} - conf.set_values(values) + values = { + k: v + for k, v in config.items() + if k not in ("config_id", "config_environment") + } + if values: + conf.set_values(values) else: session.new_config(module, environment, config) except Exception as e: if not cont_on_error: raise print(e) - print('Capsul config import fails for ' - 'environment:', environment, ', module:', - module, ', config:', config, file=sys.stderr) + print( + "Capsul config import fails for " "environment:", + environment, + ", module:", + module, + ", config:", + config, + file=sys.stderr, + ) def get_all_environments(self): - ''' + """ Get all environment values in the database - ''' + """ with self as session: return session.get_all_environments() - - + + class SettingsSession: - ''' - Settings use/modifiction session, returned by "with settings as session:" - ''' + """ + Settings use/modification session, returned by "with settings as session:" + """ - def __init__(self, populse_session, module_notifiers=None): - ''' + def __init__(self, populse_db_storage, module_notifiers=None): + """ SettingsSession are created with Settings.__enter__ using a `with` statement. - ''' - self._dbs = populse_session + """ + self._storage = populse_db_storage if module_notifiers is None: self.module_notifiers = {} else: @@ -355,101 +361,115 @@ def __init__(self, populse_session, module_notifiers=None): @staticmethod def collection_name(module): - ''' + """ Return the name of the populse_db collection corresponding to a - settings module. The result is the full name of the module + settings module. The result is the full name of the module prefixed by Settings.collection_prefix (i.e. `'settings/'`). - ''' + """ module = Settings.module_name(module) - collection = '%s%s' % (Settings.collection_prefix, module) + collection = "%s%s" % (Settings.collection_prefix, module) return collection - + def ensure_module_fields(self, module, fields): - ''' + """ Make sure that the given module exists in settings and create the given fields if they do not exist. `fields` is a list of dictionaries with three items: - name: the name of the key - type: the data type of the field (in populse_db format) - description: the documentation of the field - ''' + """ collection = self.collection_name(module) - if self._dbs.get_collection(collection) is None: - self._dbs.add_collection(collection, Settings.config_id_field) - self._dbs.add_field(collection, - Settings.environment_field, - 'string', index=True) - for field in fields: - name = field['name'] - if self._dbs.get_field(collection, name) is None: - self._dbs.add_field(collection, name=name, - field_type=field['type'], - description=field['description']) + with self._storage.schema() as schema: + schema.add_collection(collection, Settings.config_id_field) + schema.add_field(collection, Settings.environment_field, "str", index=True) + for field in fields: + name = field["name"] + schema.add_field( + collection, + name, + field_type=field["type"], + description=field["description"], + ) return collection - + def new_config(self, module, environment, values): - ''' - Creates a new configuration document for a module in the given - environment. Values is a dictionary used to set values for the - document. The document mut have a unique string identifier in + """ + Creates a new configuration document for a module in the given + environment. Values is a dictionary used to set values for the + document. The document mut have a unique string identifier in the `Settings.config_id_field` (i.e. `'config_id'`), if None is - given in `values` a unique random value is created (with + given in `values` a unique random value is created (with `uuid.uuid4()`). - ''' - document = { - Settings.environment_field: environment} + """ + document = {Settings.environment_field: environment} document.update(values) id = document.get(Settings.config_id_field) if id is None: id = str(uuid4()) - document[Settings.config_id_field] = '%s-%s' % (id, environment) collection = self.collection_name(module) - self._dbs.add_document(collection, document) + with self._storage.data(write=True) as data: + if data is not None: + data[collection][f"{id}-{environment}"] = document + else: + print("Data session is None, unable to insert document.") config = SettingsConfig( - self._dbs, collection, id, environment, - notifiers=self.module_notifiers.get(Settings.module_name(module), - [])) + self._storage, + collection, + id, + environment, + notifiers=self.module_notifiers.get(Settings.module_name(module), []), + ) config.notify() return config def remove_config(self, module, environment, config_id): - ''' + """ Removes a configuration (document in the database) for a given module / environment, identified by its `Settings.config_id_field` value. - ''' + """ collection = self.collection_name(module) - id = '%s-%s' % (config_id, environment) - self._dbs.remove_document(collection, id) + id = "%s-%s" % (config_id, environment) + with self._storage.data(write=True) as data: + del data[collection][id] def configs(self, module, environment, selection=None): - ''' + """ Returns a generator that iterates over all configuration documents created for the given module and environment. - ''' + """ collection = self.collection_name(module) - if self._dbs.get_collection(collection) is not None: - if selection: - if 'config_id' in selection: - x = selection.find('config_id') - x = selection.find('"', x + 9) - x = selection.find('"', x + 1) - selection = '%s-%s%s'% (selection[:x], environment, - selection[x:]) - full_query = '%s == "%s" AND (%s)' % ( - Settings.environment_field, environment, selection) - docs = self._dbs.filter_documents(collection, full_query) - else: - docs = self._dbs.filter_documents( - collection, - '%s=="%s"' % (Settings.environment_field, environment)) - for d in docs: - id = d[Settings.config_id_field] - id = id[:-len(environment)-1] - yield SettingsConfig( - self._dbs, collection, id, environment, - notifiers=self.module_notifiers.get(Settings.module_name( - module), [])) + with self._storage.data() as data: + if data.has_collection(collection): + if selection: + if "config_id" in selection: + x = selection.find("config_id") + x = selection.find('"', x + 9) + x = selection.find('"', x + 1) + selection = "%s-%s%s" % ( + selection[:x], + environment, + selection[x:], + ) + full_query = f'{Settings.environment_field} == "{environment}" AND ({selection})' + docs = data[collection].search(full_query) + else: + docs = data[collection].search( + f'{Settings.environment_field}=="{environment}"' + ) + for d in docs: + id = d[Settings.config_id_field] + id = id[: -len(environment) - 1] + yield SettingsConfig( + self._storage, + collection, + id, + environment, + notifiers=self.module_notifiers.get( + Settings.module_name(module), [] + ), + ) def config(self, module, environment, selection=None, any=True): - ''' + """ Selects configurations (like in :meth:`configs`) and ensures at most one one is selected @@ -471,7 +491,7 @@ def config(self, module, environment, selection=None, any=True): config: SettingsConfig instance or None None if no matching config is found or more than one if any is False - ''' + """ configs = list(self.configs(module, environment, selection)) if len(configs) == 0: return None @@ -480,69 +500,60 @@ def config(self, module, environment, selection=None, any=True): return None def get_all_environments(self): - ''' + """ Get all environment values in the database - ''' - # TODO FIXME - # this function uses low-level SQL requests on the sql engine of - # populse_db 2, because I don't know how to perform requests with the - # "DISTINCT" keyword using the high level requests language. It will - # not work using populse_db 1 nor using another (non-SQL) - # implementation of the database engine. + """ environments = set() - for collection in (i.collection_name - for i in self._dbs.get_collections()): - if collection.startswith(Settings.collection_prefix): - #collection = collection[len(Settings.collection_prefix):] - table = self._dbs.engine.collection_table[collection] - #full_query = '%s == "%s"' % ( - #Settings.environment_field, environment) - #docs = self._dbs.filter_documents(collection, full_query) - query = 'SELECT DISTINCT %s FROM "%s"' \ - % (Settings.environment_field, table) - res = self._dbs.engine.cursor.execute(query) - environments.update([r[0] for r in res]) + with self._storage.data() as data: + for collection in data.collection_names(): + if collection.startswith(Settings.collection_prefix): + rows = data[collection].search( + fields=[Settings.environment_field], as_list=True, distinct=True + ) + environments.update([row[0] for row in rows]) return environments + class SettingsConfig: - def __init__(self, populse_session, collection, id, environment, - notifiers=[]): - super(SettingsConfig, self).__setattr__('_dbs', populse_session) - super(SettingsConfig, self).__setattr__('_collection', collection) - super(SettingsConfig, self).__setattr__('_environment', environment) - super(SettingsConfig, self).__setattr__('_id', id) - super(SettingsConfig, self).__setattr__('_notifiers', notifiers) + def __init__(self, populse_session, collection, id, environment, notifiers=[]): + super(SettingsConfig, self).__setattr__("_storage", populse_session) + super(SettingsConfig, self).__setattr__("_collection", collection) + super(SettingsConfig, self).__setattr__("_environment", environment) + super(SettingsConfig, self).__setattr__("_id", id) + super(SettingsConfig, self).__setattr__("_notifiers", notifiers) def __setattr__(self, name, value): - id = '%s-%s' % (super(SettingsConfig, self).__getattribute__('_id'), - super(SettingsConfig, - self).__getattribute__('_environment')) + id = "%s-%s" % ( + super(SettingsConfig, self).__getattribute__("_id"), + super(SettingsConfig, self).__getattribute__("_environment"), + ) if getattr(self, name) != value: - self._dbs.set_value(self._collection, id, name, value) + with self._storage.data(write=True) as data: + data[self._collection][id][name] = value # notify change for listeners self.notify(name, value) def __getattr__(self, name): - if name in ('_id', '_environment'): + if name in ("_id", "_environment"): return super(SettingsConfig, self).__getattribute__(name) - value = self._dbs.get_value(self._collection, - '%s-%s' % (self._id, self._environment), - name) - if name == 'config_id': - return value[:-len(self._environment)-1] + with self._storage.data() as data: + value = data[self._collection][f"{self._id}-{self._environment}"][ + name + ].get() + if name == "config_id": + return value[: -len(self._environment) - 1] return value def set_values(self, values): - id = '%s-%s' % (self._id, self._environment) - old = self._dbs.get_document(self._collection, id, - fields=values.keys(), as_list=True) - mod_values = {k: v for o, (k, v) in zip(old, values.items()) if o != v} - if mod_values: - self._dbs.set_values(self._collection, id, mod_values) - for name, value in mod_values.items(): - self.notify(name, value) + id = "%s-%s" % (self._id, self._environment) + with self._storage.data(write=True) as data: + old = data[self._collection][id].get(fields=values.keys(), as_list=True) + mod_values = {k: v for o, (k, v) in zip(old, values.items()) if o != v} + if mod_values: + data[self._collection][id].update(mod_values) + for name, value in mod_values.items(): + self.notify(name, value) def notify(self, name=None, value=None): for notifier in self._notifiers: notifier(name, value) - diff --git a/capsul/pipeline/test/test_pipeline_workflow.py b/capsul/pipeline/test/test_pipeline_workflow.py index 05bf936dc..2de281c39 100644 --- a/capsul/pipeline/test/test_pipeline_workflow.py +++ b/capsul/pipeline/test/test_pipeline_workflow.py @@ -227,7 +227,7 @@ def test_requirements(self): self.pipeline, study_config=self.study_config) - def test_full_wf(self): + def test_full_pipeline_wf(self): engine = self.study_config.engine self.pipeline.enable_all_pipeline_steps() msgs = []