diff --git a/doc/source/fparser.rst b/doc/source/fparser.rst index f80516606..64ebefbd2 100644 --- a/doc/source/fparser.rst +++ b/doc/source/fparser.rst @@ -213,6 +213,9 @@ __ https://github.com/stfc/fparser/blob/master/src/fparser/common/readfortran.py To read Fortran code from a file, use the `FortranFileReader` class. The `FortranFileReader` class is an iterator over Fortran code lines and is derived from the `FortranReaderBase` class. + +.. autoclass:: fparser.common.readfortran.FortranReaderBase + It automatically handles line continuations and comments, as well as detecting whether a Fortran file is in free or fixed format. diff --git a/doc/source/fparser2.rst b/doc/source/fparser2.rst index 34e1de45e..d3b45f7bb 100644 --- a/doc/source/fparser2.rst +++ b/doc/source/fparser2.rst @@ -495,6 +495,23 @@ file was found but would fail if the include file was not found:: program x include 'endprogram.inc' +Compiler/OpenMP Directive support +--------------------------------- +Most Fortran compilers support directives to enable compiler-specific +functionality. Fparser has an option to support converting these into +``Directive`` nodes where possible. This option is ``process_directives``, +and by default it is set to ``False``. If its set to true, it forces +``ignore_comments`` to be ``False``. + +The supported directives are those recognized by flang, ifx, ifort (``!dir$``), +and gcc (``!gcc$``), as well as OpenMP directives (such as ``!$omp`` +or alternatives). + +For example:: + + reader = FortranFileReader("compute_mod.f90", process_directives=True) + + Preprocessing Directives ------------------------ diff --git a/src/fparser/api.py b/src/fparser/api.py index 7686ffe43..6c1b6e301 100644 --- a/src/fparser/api.py +++ b/src/fparser/api.py @@ -86,6 +86,7 @@ def get_reader( include_dirs=None, source_only=None, ignore_comments=True, + process_directives: bool = False, ): """ Returns Fortran reader instance. @@ -106,6 +107,10 @@ def get_reader( encountered. :param bool ignore_comments: Whether or not to ignore (and discard) comments when parsing the source. + :param process_directives: whether or not to process directives as + specialised Directive nodes. Default is False (in which case + directives are left as comments). This option overrides the + ignore_comments input. :returns: a reader instance :rtype: :py:class:`fparser.common.readfortran.FortranReader` @@ -138,6 +143,7 @@ def get_reader( include_dirs=include_dirs, source_only=source_only, ignore_comments=ignore_comments, + process_directives=process_directives, ) elif isinstance(source, str): reader = FortranStringReader( @@ -145,6 +151,7 @@ def get_reader( include_dirs=include_dirs, source_only=source_only, ignore_comments=ignore_comments, + process_directives=process_directives, ) else: raise TypeError("Expected string or filename input but got %s" % (type(input))) diff --git a/src/fparser/common/readfortran.py b/src/fparser/common/readfortran.py index 9fd0d7a6f..183248caf 100644 --- a/src/fparser/common/readfortran.py +++ b/src/fparser/common/readfortran.py @@ -548,20 +548,24 @@ class FortranReaderBase: code, free format Fortran code, or PYF signatures (with extended free format Fortran syntax). - :param source: a file-like object with .next() method used to \ + :param source: a file-like object with .next() method used to retrive a line. :type source: :py:class:`StringIO` or a file handle - :param mode: a FortranFormat object as returned by \ + :param mode: a FortranFormat object as returned by `sourceinfo.get_source_info()` :type mode: :py:class:`fparser.common.sourceinfo.Format` - :param bool isstrict: whether we are strictly enforcing fixed format. - :param bool ignore_comments: whether or not to discard comments. If + :param isstrict: whether we are strictly enforcing fixed format. + :param ignore_comments: whether or not to discard comments. If comments are not ignored, they will be added as special Comment node to the tree, and will therefore also be added to the output Fortran source code. - :param Optional[bool] include_omp_conditional_lines: whether or not the + :param include_omp_conditional_lines: whether or not the content of a line with an OMP sentinel is parsed or not. Default is False (in which case it is treated as a Comment). + :param process_directives: whether or not to process directives as + specialised Directive nodes. Default is False (in which case + directives are left as comments). This option overrides the + ignore_comments input. The Fortran source is iterated by `get_single_line`, `get_next_line`, `put_single_line` methods. @@ -569,7 +573,12 @@ class FortranReaderBase: """ def __init__( - self, source, mode, ignore_comments, include_omp_conditional_lines=False + self, + source, + mode: bool, + ignore_comments: bool, + include_omp_conditional_lines: bool = False, + process_directives: bool = False, ): self.source = source self._include_omp_conditional_lines = include_omp_conditional_lines @@ -579,6 +588,10 @@ def __init__( # This value for ignore_comments can be overridden by using the # ignore_comments optional argument to e.g. get_single_line() self._ignore_comments = ignore_comments + # Enabling process directives forces comments to be processed. + if process_directives: + self._ignore_comments = False + self.process_directives = process_directives self.filo_line = [] # used for un-consuming lines. self.fifo_item = [] @@ -1680,6 +1693,10 @@ class FortranFileReader(FortranReaderBase): :param Optional[bool] include_omp_conditional_lines: whether or not the content of a line with an OMP sentinel is parsed or not. Default is False (in which case it is treated as a Comment). + :param process_directives: whether or not to process directives as + specialised Directive nodes. Default is False (in which case + directives are left as comments). This option overrides the + ignore_comments input. For example:: @@ -1697,6 +1714,7 @@ def __init__( ignore_comments=True, ignore_encoding=True, include_omp_conditional_lines=False, + process_directives: bool = False, ): # The filename is used as a unique ID. This is then used to cache the # contents of the file. Obviously if the file changes content but not @@ -1729,6 +1747,7 @@ def __init__( mode, ignore_comments, include_omp_conditional_lines=include_omp_conditional_lines, + process_directives=process_directives, ) if include_dirs is None: @@ -1764,6 +1783,10 @@ class FortranStringReader(FortranReaderBase): :param Optional[bool] include_omp_conditional_lines: whether or not the content of a line with an OMP sentinel is parsed or not. Default is False (in which case it is treated as a Comment). + :param process_directives: whether or not to process directives as + specialised Directive nodes. Default is False (in which case + directives are left as comments). This option overrides the + ignore_comments input. For example: @@ -1786,6 +1809,7 @@ def __init__( ignore_comments=True, ignore_encoding=True, include_omp_conditional_lines=False, + process_directives: bool = False, ): # The Python ID of the string was used to uniquely identify it for # caching purposes. Unfortunately this ID is only unique for the @@ -1806,6 +1830,7 @@ def __init__( mode, ignore_comments, include_omp_conditional_lines=include_omp_conditional_lines, + process_directives=process_directives, ) if include_dirs is not None: self.include_dirs = include_dirs[:] diff --git a/src/fparser/common/tests/test_readfortran.py b/src/fparser/common/tests/test_readfortran.py index 1951cd46a..9d45342cd 100644 --- a/src/fparser/common/tests/test_readfortran.py +++ b/src/fparser/common/tests/test_readfortran.py @@ -1730,3 +1730,25 @@ def test_conditional_include_omp_conditional_liness_free_format_multiple(): line = reader.next() # 8 spaces in input_text, plus two for replacing the !$ assert line.line == "bla bla" + + +def test_process_directives_option_read_fortran(): + """Test handling of the process_directives option. + + Note that the funcionality tests for this option are in + fparser/two/tests/test_comments_and_directives.py""" + + input_text = "!$omp target\n! comment" + reader = FortranStringReader(input_text) + assert not reader.process_directives + assert reader._ignore_comments + + reader = FortranStringReader(input_text, process_directives=True) + assert reader.process_directives + assert not reader._ignore_comments + + reader = FortranStringReader( + input_text, ignore_comments=False, process_directives=True + ) + assert reader.process_directives + assert not reader._ignore_comments diff --git a/src/fparser/two/Fortran2003.py b/src/fparser/two/Fortran2003.py index 3fd18d710..6497dac4a 100644 --- a/src/fparser/two/Fortran2003.py +++ b/src/fparser/two/Fortran2003.py @@ -73,6 +73,8 @@ import re import sys +from typing import Union + from fparser.common.splitline import string_replace_map from fparser.two import pattern_tools as pattern from fparser.common.readfortran import FortranReaderBase @@ -116,9 +118,105 @@ # R102: = # R103: = + # # SECTION 2 # +class Directive(Base): + """ + Represents a Directive. Directives are leaves in the tree, containing + a single item consisting of the directive string. + + Fparser supports the following directive formats: + + 1. '!$dir' for generic directives. + 2. '!dir$' for the flang, ifx or ifort compilers. + 3. '!gcc$' for the gfortran compiler. + 4. '!$omp', '!$ompx', 'c$omp', '*$omp', '!$omx', 'c$omx', and '*$omx' for + OpenMP directives. + """ + + subclass_names = [] + # TODO #483 - Add OpenACC directive support. + _directive_formats = [ + "!$dir", # Generic directive + "!dir$", # flang, ifx, ifort directives. + "cdir$", # flang, ifx, ifort fixed format directive. + "!$omp", # OpenMP directive + "c$omp", # OpenMP fixed format directive + "*$omp", # OpenMP fixed format directive + "!$omx", # OpenMP fixed format directive + "c$omx", # OpenMP fixed format directive + "*$omx", # OpenMP fixed format directive + "!gcc$", # GCC compiler directive + "!$ompx", # OpenMP extension directive + ] + + @show_result + def __new__(cls, string: Union[str, FortranReaderBase], parent_cls=None): + """ + Create a new Directive instance. + + :param type cls: the class of object to create. + :param string: (source of) Fortran string to parse. + :param parent_cls: the parent class of this object. + :type parent_cls: :py:type:`type` + + """ + from fparser.common import readfortran + + if isinstance(string, readfortran.Comment): + # Directives must start with one of the specified directive + # prefixes. + lower = string.comment.lower() + if not ( + any( + [ + lower.startswith(prefix) + for prefix in Directive._directive_formats + ] + ) + ): + return + # We were after a directive and we got a directive. Construct + # one manually to avoid recursively calling this __new__ + # method again... + obj = object.__new__(cls) + obj.init(string) + return obj + if isinstance(string, FortranReaderBase): + reader = string + item = reader.get_item() + if item is None: + return + if isinstance(item, readfortran.Comment): + # This effectively recursively calls this routine + res = Directive(item) + if not res: + # We didn't get a directive so put the item back in + # the FIFO + reader.put_item(item) + return res + # We didn't get a directive so put the item back in the FIFO + reader.put_item(item) + # We didn't get a directive + return + + def init(self, comment) -> None: + """ + Initialise this Directive from a comment object. + + :param comment: The comment object produced by the reader + :type comment: :py:class:`readfortran.Comment` + """ + self.items = [comment.comment] + self.item = comment + + def tostr(self) -> str: + """ + :returns: this directive as a string. + """ + return str(self.items[0]) class Comment(Base): @@ -182,32 +280,28 @@ def tostr(self): """ return str(self.items[0]) - def restore_reader(self, reader): - """ - Undo the read of this comment by putting its content back - into the reader (which has a FIFO buffer) - - :param reader: the reader instance to return the comment to - :type reader: :py:class:`fparser.readfortran.FortranReaderBase` - """ - reader.put_item(self.item) - def match_comment_or_include(reader): - """Creates a comment or include object from the current line. + """Creates a comment, directive, or include object from the current line. - :param reader: the fortran file reader containing the line \ + :param reader: the fortran file reader containing the line of code that we are trying to match - :type reader: :py:class:`fparser.common.readfortran.FortranFileReader` \ - or \ - :py:class:`fparser.common.readfortran.FortranStringReader` + :type reader: :py:class:`fparser.common.readfortran.FortranFileReader` + or + :py:class:`fparser.common.readfortran.FortranStringReader` - :return: a comment or include object if found, otherwise `None`. - :rtype: :py:class:`fparser.two.Fortran2003.Comment` or \ + :return: a comment, directive, or include object if found, otherwise + `None`. + :rtype: :py:class:`fparser.two.Fortran2003.Comment` or :py:class:`fparser.two.Fortran2003.Include_Stmt` + or :py:class:`fparser.two.Fortran2003.Directive` """ - obj = Comment(reader) + obj = None + # Whether or not to specialise Directives is a run-time option. + if reader.process_directives: + obj = Directive(reader) + obj = Comment(reader) if not obj else obj obj = Include_Stmt(reader) if not obj else obj return obj diff --git a/src/fparser/two/tests/test_comments.py b/src/fparser/two/tests/test_comments_and_directives.py similarity index 77% rename from src/fparser/two/tests/test_comments.py rename to src/fparser/two/tests/test_comments_and_directives.py index 29bc15d7e..120a4444b 100644 --- a/src/fparser/two/tests/test_comments.py +++ b/src/fparser/two/tests/test_comments_and_directives.py @@ -34,7 +34,7 @@ """Module containing tests for aspects of fparser2 related to comments""" import pytest -from fparser.two.Fortran2003 import Program, Comment, Subroutine_Subprogram +from fparser.two.Fortran2003 import Program, Comment, Directive, Subroutine_Subprogram from fparser.two.utils import walk from fparser.api import get_reader @@ -409,3 +409,125 @@ def test_action_stmts(): assert "a big array" in str(ifstmt) cmt = get_child(ifstmt, Comment) assert cmt.parent is ifstmt + + +def test_directive_stmts(): + """Test that directives are created instead of comments when + appropriate.""" + source = """ + Program my_prog + integer :: x !$dir inline + + !dir$ compiler directive + !$omp target + !$omp loop + do x= 1 , 100 + ! A comment! + !!$ Another comment + end do + End Program""" + reader = get_reader( + source, isfree=True, ignore_comments=False, process_directives=True + ) + program = Program(reader) + out = walk(program, Directive) + assert len(out) == 4 + assert out[0].items[0] == "!$dir inline" + assert out[1].items[0] == "!dir$ compiler directive" + assert out[2].items[0] == "!$omp target" + assert out[3].items[0] == "!$omp loop" + + assert out[3].tostr() == "!$omp loop" + + # Check the restore_reader works correctly for directive. + old = reader.get_item() + assert old == None + out[2].restore_reader(reader) + old = reader.get_item() + assert old is not None + + out = walk(program, Comment) + comments = 0 + for comment in out: + if comment.items[0] != "": + comments = comments + 1 + assert comments == 2 + assert str(out[2]) == "! A comment!" + assert str(out[3]) == "!!$ Another comment" + + # Check that passing something that isn't a comment into a Directive + # __new__ call doesn't create a Directive. + out = Directive(program) + assert out is None + + # Fixed form test that directives are handled. + reader = get_reader( + """\ + program foo +cdir$ This is a directive +C This is a comment +C$ integer omp_get_thread_num + end program foo""", + isfree=False, + ignore_comments=False, + process_directives=True, + ) + program = Program(reader) + out = walk(program, Directive) + assert len(out) == 1 + assert out[0].items[0] == "cdir$ This is a directive" + + +@pytest.mark.parametrize( + "directive,expected,free", + [ + ("!$dir always", "!$dir always", True), + ("!dir$ always", "!dir$ always", True), + ("!gcc$ vector", "!gcc$ vector", True), + ("!$omp parallel", "!$omp parallel", True), + ("!$ompx parallel", "!$ompx parallel", True), + ("c$omp parallel", "c$omp parallel", False), + ("c$omx parallel", "c$omx parallel", False), + ("!$omx parallel", "!$omx parallel", False), + ("*$omp parallel", "*$omp parallel", False), + ("c$omx parallel", "c$omx parallel", False), + ("*$omx parallel", "*$omx parallel", False), + ], +) +def test_all_directive_formats(directive, expected, free): + """Parameterized test to ensure that all directive formats are + correctly recognized.""" + # Tests for free-form directives + if free: + source = """ + Program my_prog + integer :: x + """ + source = source + directive + "\n" + source = ( + source + + """ do x= 1 , 100 + end do + End Program""" + ) + else: + source = """\ + program foo +""" + source = source + directive + "\n" + source = source + " end program foo" + + reader = get_reader( + source, isfree=free, ignore_comments=False, process_directives=True + ) + program = Program(reader) + out = walk(program, Directive) + assert len(out) == 1 + assert out[0].items[0] == expected + + # Test that we correctly get directives without ignore_comments=False. + reader = get_reader(source, isfree=free, process_directives=True) + program = Program(reader) + out = walk(program, Directive) + assert len(out) == 1 + assert out[0].items[0] == expected diff --git a/src/fparser/two/utils.py b/src/fparser/two/utils.py index 3c562e5c1..44e98e066 100644 --- a/src/fparser/two/utils.py +++ b/src/fparser/two/utils.py @@ -331,6 +331,7 @@ def import_now(): End_Select_Type_Stmt, Case_Stmt, End_Select_Stmt, + Directive, Comment, Include_Stmt, add_comments_includes_directives, @@ -347,6 +348,7 @@ def import_now(): DynamicImport.End_Select_Type_Stmt = End_Select_Type_Stmt DynamicImport.Case_Stmt = Case_Stmt DynamicImport.End_Select_Stmt = End_Select_Stmt + DynamicImport.Directive = Directive DynamicImport.Comment = Comment DynamicImport.Include_Stmt = Include_Stmt DynamicImport.C99Preprocessor = C99Preprocessor @@ -703,8 +705,12 @@ def match( if match_names: start_name = obj.get_start_name() - # Comments and Include statements are always valid sub-classes - classes = subclasses + [di.Comment, di.Include_Stmt] + # Directives, Comments and Include statements are always valid sub-classes + comments = [di.Comment, di.Include_Stmt] + # Only add directives if enabled. + if reader.process_directives: + comments.insert(0, di.Directive) + classes = subclasses + comments # Preprocessor directives are always valid sub-classes cpp_classes = [ getattr(di.C99Preprocessor, cls_name)