Driver-specific Loki pipelines

cmake/loki_transform.cmake

@@ -42,7 +42,7 @@ include( loki_transform_helpers )
 
 function( loki_transform )
 
-    set( options CPP )
+    set( options CPP MULTIMODE )
     set( oneValueArgs COMMAND MODE FRONTEND CONFIG BUILDDIR )
     set( multiValueArgs OUTPUT DEPENDS SOURCES HEADERS INCLUDES DEFINITIONS OMNI_INCLUDE XMOD )
 
@@ -73,6 +73,10 @@ function( loki_transform )
         list( APPEND _ARGS --cpp )
     endif()
 
+    if ( _PAR_MULTIMODE )
+        list( APPEND _ARGS --multimode )
+    endif()
+
     # Ensure transformation script and environment is available
     _loki_transform_env_setup()
 
@@ -115,7 +119,7 @@ endfunction()
 
 function( loki_transform_plan )
 
-    set( options NO_SOURCEDIR CPP )
+    set( options NO_SOURCEDIR CPP MULTIMODE )
     set( oneValueArgs MODE FRONTEND CONFIG BUILDDIR SOURCEDIR CALLGRAPH PLAN )
     set( multiValueArgs SOURCES HEADERS )
 
@@ -152,6 +156,10 @@ function( loki_transform_plan )
         ecbuild_critical( "No PLAN file specified for loki_transform_plan()" )
     endif()
 
+    if ( _PAR_MULTIMODE )
+        list( APPEND _ARGS --multimode )
+    endif()
+
     _loki_transform_env_setup()
 
     # Create a source transformation plan to tell CMake which files will be affected
@@ -300,7 +308,7 @@ endfunction()
 
 function( loki_transform_target )
 
-    set( options NO_PLAN_SOURCEDIR COPY_UNMODIFIED CPP CPP_PLAN )
+    set( options NO_PLAN_SOURCEDIR COPY_UNMODIFIED CPP CPP_PLAN MULTIMODE )
     set( single_value_args COMMAND MODE FRONTEND CONFIG PLAN )
     set( multi_value_args TARGET SOURCES HEADERS DEFINITIONS INCLUDES )
 
@@ -334,6 +342,9 @@ function( loki_transform_target )
     if( _PAR_T_CPP_PLAN )
         list( APPEND _PLAN_OPTIONS CPP )
     endif()
+    if ( _PAR_T_MULTIMODE )
+        list( APPEND _PLAN_OPTIONS MULTIMODE )
+    endif()
     if( _PAR_T_NO_PLAN_SOURCEDIR )
         list( APPEND _PLAN_OPTIONS NO_SOURCEDIR )
     endif()
@@ -367,6 +378,9 @@ function( loki_transform_target )
         if( _PAR_T_CPP )
             list( APPEND _TRANSFORM_OPTIONS CPP )
         endif()
+        if ( _PAR_T_MULTIMODE )
+            list( APPEND _TRANSFORM_OPTIONS MULTIMODE )
+        endif()
 
         loki_transform(
             COMMAND     ${_PAR_T_COMMAND}

loki/batch/configure.py

@@ -432,6 +432,13 @@ def mode(self):
         """
         return self.config.get('mode', None)
 
+    @property
+    def inherited_mode(self):
+        """
+        Transformation "inherited_mode" for multi-mode processing
+        """
+        return self.config.get('inherited_mode', None)
+
     @property
     def expand(self):
         """

loki/batch/item.py

@@ -564,6 +564,14 @@ def path(self):
         """
         return self.source.path
 
+    @property
+    def orig_path(self):
+        """
+        The original filepath of the associated source file
+        necessary for planning when duplicating/renaming items
+        """
+        return self.source.orig_path
+
 
 class FileItem(Item):
     """

loki/batch/item_factory.py

@@ -290,6 +290,7 @@ def get_or_create_item_from_item(self, name, item, config=None):
 
             # Create a new FileItem for the new source
             new_source.path = item.path.with_name(f'{scope_name or local_name}{item.path.suffix}')
+            new_source.orig_path = item.path
             file_item = self.get_or_create_file_item_from_source(new_source, config=config)
 
             # Get the definition items for the FileItem and return the new item
@@ -346,6 +347,13 @@ def get_or_create_file_item_from_path(self, path, config, frontend_args=None):
         self.item_cache[item_name] = file_item
         return file_item
 
+    def get_file_item_from_source(self, source):
+        # Check for file item with the same source object
+        for item in self.item_cache.values():
+            if isinstance(item, FileItem) and item.source is source:
+                return item
+        return None
+
     def get_or_create_file_item_from_source(self, source, config):
         """
         Utility method to create a :any:`FileItem` corresponding to a given source object
@@ -368,9 +376,9 @@ def get_or_create_file_item_from_source(self, source, config):
             The config object from which the item configuration will be derived
         """
         # Check for file item with the same source object
-        for item in self.item_cache.values():
-            if isinstance(item, FileItem) and item.source is source:
-                return item
+        item_ = self.get_file_item_from_source(source)
+        if item_ is not None:
+            return item_
 
         if not source.path:
             raise RuntimeError('Cannot create FileItem from source: Sourcefile has no path')

loki/batch/scheduler.py

@@ -5,6 +5,7 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
+import sys
 from enum import Enum, auto
 from os.path import commonpath
 from pathlib import Path
@@ -202,6 +203,38 @@ def __init__(self, paths, config=None, seed_routines=None, preprocess=False,
             # Attach interprocedural call-tree information
             self._enrich()
 
+    def propagate_and_separate_modes(self, proc_strategy=ProcessingStrategy.DEFAULT):
+        from loki.transformations.dependency import SeparateModesKernel # pylint: disable=import-outside-toplevel
+        self._propagate_modes()
+        self.process_transformation(SeparateModesKernel(), proc_strategy=proc_strategy)
+        self._propagate_modes_set()
+        modes = {item.mode for item in self.items}
+        return as_tuple(modes)
+
+
+    def _propagate_modes(self):
+        driver_items = [item for item in self.items if item.role == 'driver']
+        for item in driver_items:
+            module_file_items = self.sgraph.get_corresponding_module_and_file_item(item, self.item_factory)
+            for _item in module_file_items:
+                if _item is not None:
+                    _item.config['mode'] = item.mode
+            descendants = self.sgraph.descendants(item, self.item_factory,
+                    include_module_items=True, include_file_items=True)
+            for descendant in descendants:
+                if descendant is not None:
+                    descendant.config.setdefault('inherited_mode', set()).add(item.mode)
+
+    def _propagate_modes_set(self):
+        driver_items = [item for item in self.items if item.role == 'driver']
+        for item in driver_items:
+            descendants = self.sgraph.descendants(item, self.item_factory,
+                    include_module_items=True, include_file_items=True)
+            for descendant in descendants:
+                if descendant is not None:
+                    descendant.config['mode'] = item.mode
+                    descendant.config['inherited_mode'] = set()
+
     @Timer(logger=info, text='[Loki::Scheduler] Performed initial source scan in {:.2f}s')
     def _discover(self):
         """
@@ -429,7 +462,7 @@ def process(self, transformation, proc_strategy=ProcessingStrategy.DEFAULT):
 
         Parameters
         ----------
-        transformation : :any:`Transformation` or :any:`Pipeline`
+        transformation : :any:`Transformation` or :any:`Pipeline` or dict of :any:`Pipeline`s
             The transformation or transformation pipeline to apply
         proc_strategy : :any:`ProcessingStrategy`
             The processing strategy to use when applying the given
@@ -441,13 +474,24 @@ def process(self, transformation, proc_strategy=ProcessingStrategy.DEFAULT):
         elif isinstance(transformation, Pipeline):
             self.process_pipeline(pipeline=transformation, proc_strategy=proc_strategy)
 
+        elif isinstance(transformation, dict):
+            assert all(isinstance(trafo, Pipeline) for trafo in transformation.values())
+            modes = self.propagate_and_separate_modes(proc_strategy=proc_strategy)
+            # check if all required pipelines are available
+            for mode in modes:
+                if mode not in transformation:
+                    msg = f'[Loki] ERROR: Pipeline or transformation mode {mode} not found in config file.\n'
+                    sys.exit(msg)
+            # apply those pipelines
+            for mode in modes:
+                self.process_pipeline(pipeline=self.config.pipelines[mode], proc_strategy=proc_strategy, mode=mode)
         else:
             error('[Loki::Scheduler] Batch processing requires Transformation or Pipeline object')
             raise RuntimeError(f'Could not batch process {transformation}')
 
-    def process_pipeline(self, pipeline, proc_strategy=ProcessingStrategy.DEFAULT):
+    def process_pipeline(self, pipeline, proc_strategy=ProcessingStrategy.DEFAULT, mode=None):
         """
-        Process a given :any:`Pipeline` by applying its assocaited
+        Process a given :any:`Pipeline` by applying its associated
         transformations in turn.
 
         Parameters
@@ -457,11 +501,14 @@ def process_pipeline(self, pipeline, proc_strategy=ProcessingStrategy.DEFAULT):
         proc_strategy : :any:`ProcessingStrategy`
             The processing strategy to use when applying the given
             :data:`pipeline` to the scheduler's graph.
+        mode : str, optional
+            Transformation mode, selecting which code transformations/pipeline on which graph to apply.
+            Default: `None`, thus mode agnostic.
         """
         for transformation in pipeline.transformations:
-            self.process_transformation(transformation, proc_strategy=proc_strategy)
+            self.process_transformation(transformation, proc_strategy=proc_strategy, mode=mode)
 
-    def process_transformation(self, transformation, proc_strategy=ProcessingStrategy.DEFAULT):
+    def process_transformation(self, transformation, proc_strategy=ProcessingStrategy.DEFAULT, mode=None):
         """
         Process all :attr:`items` in the scheduler's graph
 
@@ -489,6 +536,9 @@ def process_transformation(self, transformation, proc_strategy=ProcessingStrateg
         proc_strategy : :any:`ProcessingStrategy`
             The processing strategy to use when applying the given
             :data:`transformation` to the scheduler's graph.
+        mode : str, optional
+            Transformation mode, selecting which code transformations on which graph to apply.
+            Default: `None`, thus mode agnostic.
         """
         def _get_definition_items(_item, sgraph_items):
             # For backward-compatibility with the DependencyTransform and LinterTransformation
@@ -527,15 +577,17 @@ def _get_definition_items(_item, sgraph_items):
                 sgraph_items = sgraph.items
                 traversal = SFilter(
                     graph, reverse=transformation.reverse_traversal,
-                    include_external=self.config.default.get('strict', True)
+                    include_external=self.config.default.get('strict', True),
+                    mode=mode
                 )
             else:
                 graph = self.sgraph
                 sgraph_items = graph.items
                 traversal = SFilter(
                     graph, item_filter=item_filter, reverse=transformation.reverse_traversal,
                     exclude_ignored=not transformation.process_ignored_items,
-                    include_external=self.config.default.get('strict', True)
+                    include_external=self.config.default.get('strict', True),
+                    mode=mode
                 )
 
             # Collect common transformation arguments

loki/batch/sfilter.py

@@ -7,7 +7,7 @@
 
 import networkx as nx
 
-from loki.batch.item import Item, ExternalItem
+from loki.batch.item import Item, ExternalItem, TypeDefItem, InterfaceItem
 
 
 __all__ = ['SFilter']
@@ -38,9 +38,12 @@ class SFilter:
         Exclude :any:`Item` objects that have the ``is_ignored`` property
     include_external : bool, optional
         Do not skip :any:`ExternalItem` in the iterator
+    mode : str, optional
+        Only include items having corresponding mode.
     """
 
-    def __init__(self, sgraph, item_filter=None, reverse=False, exclude_ignored=False, include_external=False):
+    def __init__(self, sgraph, item_filter=None, reverse=False, exclude_ignored=False, include_external=False,
+            mode=None):
         self.sgraph = sgraph
         self.reverse = reverse
         if item_filter:
@@ -49,6 +52,7 @@ def __init__(self, sgraph, item_filter=None, reverse=False, exclude_ignored=Fals
             self.item_filter = Item
         self.exclude_ignored = exclude_ignored
         self.include_external = include_external
+        self.mode = mode
 
     def __iter__(self):
         if self.reverse:
@@ -68,5 +72,10 @@ def __next__(self):
                 node_cls = type(node)
             if issubclass(node_cls, self.item_filter) and not (self.exclude_ignored and node.is_ignored):
                 # We found the next item matching the filter (and which is not ignored, if applicable)
-                break
+                if self.mode is None:
+                    break
+                if isinstance(node, (ExternalItem, TypeDefItem, InterfaceItem)):
+                    break
+                if node.mode == self.mode:
+                    break
         return node

loki/batch/sgraph.py

@@ -491,3 +491,44 @@ def export_to_file(self, dotfile_path):
             graph.render(path, view=False)
         except gviz.ExecutableNotFound as e:
             warning(f'[Loki] Failed to render callgraph due to graphviz error:\n  {e}')
+
+    def descendants(self, item, item_factory, include_module_items=False, include_file_items=False):
+        """
+        Get all descendants of a given :data:`item`.
+
+        Parameters
+        ----------
+        item : :any:`Item`
+            The item node in the dependency graph for which to determine the successors
+        item_factory : :any:`ItemFactory`
+            The item factory to use.
+        include_module_items : bool, optional
+            Whether to include module items.
+        include_file_items : bool, optional
+            Whether to include file items.
+        """
+        item_descendants = list(nx.descendants(self._graph, item))
+        module_items = []
+        file_items = []
+        if include_module_items:
+            module_items = [item_factory.item_cache.get(_item.scope_name)
+                    for _item in item_descendants if not _item.is_ignored]
+        if include_file_items:
+            file_items = [item_factory.get_file_item_from_source(_item.source)
+                    for _item in item_descendants if not _item.is_ignored]
+        return item_descendants + module_items + file_items
+
+    def get_corresponding_module_and_file_item(self, item, item_factory):
+        """
+        Get corresponding module and file item of a given :data:`item`.
+
+        Parameters
+        ----------
+        item : :any:`Item`
+            The item node in the dependency graph for which to determine the successors
+        item_factory : :any:`ItemFactory`
+            The item factory to use.
+        """
+        _items = (item_factory.item_cache.get(item.scope_name),)
+        _items += (item_factory.get_file_item_from_source(item.source),)
+        return _items