(Towards #2384) directive codeblocks

doc/user_guide/psyclone_command.rst

@@ -54,7 +54,8 @@ by the command:
     usage: psyclone [-h] [-v] [-c CONFIG] [-s SCRIPT] [-I INCLUDE] [-l {off,all,output}] [-p {invokes,routines,kernels}]
                     [--backend {enable-validation,disable-validation}]
                     [-o OUTPUT_FILE] [-api DSL] [-oalg OUTPUT_ALGORITHM_FILE] [-opsy OUTPUT_PSY_FILE] [-okern OUTPUT_KERNEL_PATH] [-d DIRECTORY] [-dm] [-nodm]
-                    [--kernel-renaming {multiple,single}] [--log-level {OFF,DEBUG,INFO,WARNING,ERROR,CRITICAL}] [--log-file LOG_FILE]
+                    [--kernel-renaming {multiple,single}] [--log-level {OFF,DEBUG,INFO,WARNING,ERROR,CRITICAL}]
+                    [--log-file LOG_FILE] [--keep-comments] [--keep-directives]
                     filename
 
     Transform a file using the PSyclone source-to-source Fortran compiler
@@ -103,6 +104,9 @@ by the command:
      --log-level {OFF,DEBUG,INFO,WARNING,ERROR,CRITICAL}
                            sets the level of the logging (defaults to OFF).
      --log-file LOG_FILE   sets the output file to use for logging (defaults to stderr).
+     --keep-comments       keeps comments from the original code (defaults to False).
+                           Directives are not kept with this option (use --keep-directives).
+     --keep-directives     keeps directives from the original code (defaults to False).
 
 Basic Use
 ---------
@@ -438,3 +442,20 @@ By default the output from the logging goes into stderr.
 To control the logging output, PSyclone provides the
 ``--log-file`` option. If this is set, the logging output will instead
 be directed to the provided file.
+
+Keeping Comments and Directives
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+PSyclone can now keep comments and directives from the original code, with
+some limitations:
+
+  1. Comments that appear after all statements in a routine are not currently
+     kept.
+  2. Directives are kept as ``CodeBlock`` nodes in the PSyIR which means
+     some transformations will be unavailable on regions containing these
+     nodes. Also PSyclone will not know any details about these nodes
+     (including that they contain directives) but this functionality will
+     be improved over time.
+
+Note that using the ``keep-comments`` option alone means that any comments
+that PSyclone interprets as directives will be lost from the input.

examples/gocean/eg6/Makefile

@@ -42,4 +42,4 @@ ENV = PSYCLONE_CONFIG=${PSYCLONE_DIR}/config/psyclone.cfg
 
 transform:
 	${PSYCLONE} -nodm -s ./backends_transform.py -api gocean alg.f90 \
-		-oalg /dev/null -opsy /dev/null
+		-oalg /dev/null -opsy /dev/null --keep-comments

examples/lfric/eg17/full_example/Makefile

@@ -87,7 +87,7 @@ main_alg.f90: main_psy.f90
 transform: main_psy.f90
 
 %_psy.f90:	%.x90
-	${PSYCLONE} -api lfric -opsy $*_psy.f90 -oalg $*_alg.f90 $<
+	${PSYCLONE} -api lfric --keep-comments --keep-directives -opsy $*_psy.f90 -oalg $*_alg.f90 $<
 
 allclean: clean
 	make -C $(LFRIC_PATH) allclean

examples/nemo/eg5/Makefile

@@ -92,7 +92,7 @@ transform: kernels
 # Need `-l all` to ensure line-lengths in generated code are less than the
 # standard-mandated 132 chars.
 kernels: extract_kernels.py
-	$(PSYCLONE) -l all -s ./extract_kernels.py -o psy.f90 ../code/tra_adv.F90
+	$(PSYCLONE) --keep-comments --keep-directives -l all -s ./extract_kernels.py -o psy.f90 ../code/tra_adv.F90
 
 $(EXTRACT_DIR)/$(LIB_NAME):
 	${MAKE} -C $(EXTRACT_DIR)

src/psyclone/domain/common/transformations/alg_invoke_2_psy_call_trans.py

@@ -231,6 +231,10 @@ def apply(self, node, options=None):
             interface=interface)
 
         psy_call = Call.create(routine_symbol, arguments)
+        # Copy over the comments.
+        psy_call.preceding_comment = node.preceding_comment
+        psy_call.inline_comment = node.inline_comment
+
         node.replace_with(psy_call)
 
         # Remove original 'invoke' symbol if there are no other

src/psyclone/domain/common/transformations/raise_psyir_2_alg_trans.py

@@ -267,6 +267,11 @@ def apply(self, call, index, options=None):
 
         invoke_call = AlgorithmInvokeCall.create(
             call.routine.symbol, calls, index, name=call_name)
+
+        # Keep comments
+        invoke_call.preceding_comment = call.preceding_comment
+        invoke_call.inline_comment = call.inline_comment
+
         call.replace_with(invoke_call)
 
 

src/psyclone/domain/lfric/transformations/raise_psyir_2_lfric_alg_trans.py

@@ -104,6 +104,11 @@ def apply(self, call, index, options=None):
 
         invoke_call = LFRicAlgorithmInvokeCall.create(
             call.routine.symbol, calls, index, name=call_name)
+
+        # Copy across any comments.
+        invoke_call.preceding_comment = call.preceding_comment
+        invoke_call.inline_comment = call.inline_comment
+
         call.replace_with(invoke_call)
 
 

src/psyclone/generator.py

@@ -35,6 +35,7 @@
 # Modified A. J. Voysey, Met Office
 # Modified J. Henrichs, Bureau of Meteorology
 # Modified A. R. Pirrie, Met Office
+# Modified A. B. G. Chalk, STFC Daresbury Lab
 
 '''
     This module provides the PSyclone 'main' routine which is intended
@@ -175,7 +176,9 @@ def generate(filename, api="", kernel_paths=None, script_name=None,
              line_length=False,
              distributed_memory=None,
              kern_out_path="",
-             kern_naming="multiple"):
+             kern_naming="multiple",
+             keep_comments: bool = False,
+             keep_directives: bool = False):
     # pylint: disable=too-many-arguments, too-many-statements
     # pylint: disable=too-many-branches, too-many-locals
     '''Takes a PSyclone algorithm specification as input and outputs the
@@ -209,6 +212,9 @@ def generate(filename, api="", kernel_paths=None, script_name=None,
     :rtype: Tuple[:py:class:`fparser.one.block_statements.BeginSource`,
         :py:class:`fparser.one.block_statements.Module`] |
         Tuple[:py:class:`fparser.one.block_statements.BeginSource`, str]
+    :param keep_comments: whether to keep comments from the original source.
+    :param keep_directives: whether to keep directives from the original
+                            source.
 
     :raises GenerationError: if an invalid API is specified.
     :raises GenerationError: if an invalid kernel-renaming scheme is specified.
@@ -274,17 +280,19 @@ def generate(filename, api="", kernel_paths=None, script_name=None,
 
     elif api in GOCEAN_API_NAMES or (api in LFRIC_API_NAMES and LFRIC_TESTING):
         # Create language-level PSyIR from the Algorithm file
-        reader = FortranReader()
+        reader = FortranReader(ignore_comments=not keep_comments,
+                               ignore_directives=not keep_directives)
         if api in LFRIC_API_NAMES:
             # avoid undeclared builtin errors in PSyIR by adding a
             # wildcard use statement.
-            fp2_tree = parse_fp2(filename)
+            fp2_tree = parse_fp2(filename, ignore_comments=not keep_comments)
             # Choose a module name that is invalid Fortran so that it
             # does not clash with any existing names in the algorithm
             # layer.
             builtins_module_name = "_psyclone_builtins"
             add_builtins_use(fp2_tree, builtins_module_name)
-            psyir = Fparser2Reader().generate_psyir(fp2_tree)
+            psyir = Fparser2Reader(ignore_directives=not keep_directives).\
+                generate_psyir(fp2_tree)
             # Check that there is only one module/program per file.
             check_psyir(psyir, filename)
         else:
@@ -512,6 +520,16 @@ def main(arguments):
         "--log-file", default=None,
         help="sets the output file to use for logging (defaults to stderr)."
     )
+    parser.add_argument(
+        "--keep-comments", default=False, action="store_true",
+        help="keeps comments from the original code (defaults to False). "
+             "Directives are not kept with this option (use "
+             "--keep-directives)."
+    )
+    parser.add_argument(
+        "--keep-directives", default=False, action="store_true",
+        help="keeps directives from the original code (defaults to False)."
+    )
 
     args = parser.parse_args(arguments)
 
@@ -588,10 +606,17 @@ def main(arguments):
         print(str(err), file=sys.stderr)
         sys.exit(1)
 
+    if args.keep_directives and not args.keep_comments:
+        logger.warning("keep_directives requires keep_comments so "
+                       "PSyclone enabled keep_comments.")
+        args.keep_comments = True
+
     if not args.psykal_dsl:
         code_transformation_mode(input_file=args.filename,
                                  recipe_file=args.script,
                                  output_file=args.o,
+                                 keep_comments=args.keep_comments,
+                                 keep_directives=args.keep_directives,
                                  line_length=args.limit)
     else:
         # PSyKAl-DSL mode
@@ -619,7 +644,9 @@ def main(arguments):
                                 line_length=(args.limit == 'all'),
                                 distributed_memory=args.dist_mem,
                                 kern_out_path=kern_out_path,
-                                kern_naming=args.kernel_renaming)
+                                kern_naming=args.kernel_renaming,
+                                keep_comments=args.keep_comments,
+                                keep_directives=args.keep_directives)
         except NoInvokesError:
             _, exc_value, _ = sys.exc_info()
             print(f"Warning: {exc_value}")
@@ -729,6 +756,7 @@ def add_builtins_use(fp2_tree, name):
 
 
 def code_transformation_mode(input_file, recipe_file, output_file,
+                             keep_comments: bool, keep_directives: bool,
                              line_length="off"):
     ''' Process the input_file with the recipe_file instructions and
     store it in the output_file.
@@ -743,6 +771,9 @@ def code_transformation_mode(input_file, recipe_file, output_file,
     :type input_file: Optional[str | os.PathLike]
     :param output_file: the output file where to store the resulting code.
     :type output_file: Optional[str | os.PathLike]
+    :param keep_comments: whether to keep comments from the original source.
+    :param keep_directives: whether to keep directives from the original
+                            source.
     :param str line_length: set to "output" to break the output into lines
         of 123 chars, and to "all", to additionally check the input code.
 
@@ -768,7 +799,9 @@ def code_transformation_mode(input_file, recipe_file, output_file,
                     sys.exit(1)
 
         # Parse file
-        psyir = FortranReader(resolve_modules=resolve_mods)\
+        psyir = FortranReader(resolve_modules=resolve_mods,
+                              ignore_comments=not keep_comments,
+                              ignore_directives=not keep_directives)\
             .psyir_from_file(input_file)
 
         # Modify file

src/psyclone/parse/utils.py

@@ -32,6 +32,7 @@
 # POSSIBILITY OF SUCH DAMAGE.
 # -----------------------------------------------------------------------------
 # Authors: R. W. Ford, A. R. Porter, S. Siso and N. Nobre, STFC Daresbury Lab
+# Modified: A. B. G. Chalk, STFC Daresbury Lab
 
 '''Utility module containing classes and functions that are used by
 the parser modules.
@@ -104,11 +105,13 @@ def check_line_length(filename):
             f"'-l/--limit' setting on the PSyclone command line.")
 
 
-def parse_fp2(filename):
+def parse_fp2(filename, ignore_comments: bool = True):
     '''Parse a Fortran source file contained in the file 'filename' using
     fparser2.
 
     :param str filename: source file (including path) to read.
+    :param ignore_comments: whether to remove the comments from the input
+                           file. Default is True.
     :returns: fparser2 AST for the source file.
     :rtype: :py:class:`fparser.two.Fortran2003.Program`
     :raises ParseError: if the file could not be parsed.
@@ -118,7 +121,8 @@ def parse_fp2(filename):
     # our configuration object.
     config = Config.get()
     try:
-        reader = FortranFileReader(filename, include_dirs=config.include_paths)
+        reader = FortranFileReader(filename, include_dirs=config.include_paths,
+                                   ignore_comments=ignore_comments)
     except IOError as error:
         raise ParseError(
             f"algorithm.py:parse_fp2: Failed to parse file '{filename}'. "