499 avoid exponential loop handling

src/fparser/two/C99Preprocessor.py

@@ -32,7 +32,11 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-"""C99 Preprocessor Syntax Rules."""
+"""C99 Preprocessor Syntax Rules. It also supports linemarker statements
+(which are technically not preprocessor directives, but are very close
+in their syntax, i.e. starting with `#`)
+
+"""
 
 # Author: Balthasar Reuter <balthasar.reuter@ecmwf.int>
 # Based on previous work by Martin Schlipf (https://github.com/martin-schlipf)
@@ -57,6 +61,7 @@
     "Cpp_Macro_Stmt",
     "Cpp_Undef_Stmt",
     "Cpp_Line_Stmt",
+    "Cpp_Linemarker_Stmt",
     "Cpp_Error_Stmt",
     "Cpp_Warning_Stmt",
     "Cpp_Null_Stmt",
@@ -649,6 +654,55 @@ def tostr(self):
         return "{0} {1}".format(*self.items)
 
 
+class Cpp_Linemarker_Stmt(WORDClsBase):  # Linemarker
+    """
+    Linemarker
+
+    linemarker-stmt is # digit-sequence "s-char-sequence" [digit ...]
+    """
+
+    subclass_names = []
+    use_names = ["Cpp_Pp_Tokens"]
+
+    # The match method will check that it is a valid linemarker, i.e.
+    # it has a line number, and file name in double quotes.
+    _pattern = pattern.Pattern("<linemarker>", r"^\s*#\s+\d+\s+\".*\".*$")
+
+    @staticmethod
+    def match(string):
+        """Implements the matching for a linemarker.
+        The right hand side of the directive is not matched any further
+        but simply kept as a string.
+
+        :param str string: the string to match with as a line statement.
+
+        :return: a tuple of size 1 with the right hand side as a string, \
+                  or `None` if there is no match.
+        :rtype: (`str`) or `NoneType`
+
+        """
+        if not string:
+            return None
+
+        return WORDClsBase.match(
+            Cpp_Linemarker_Stmt._pattern,
+            Cpp_Pp_Tokens,
+            string,
+            colons=False,
+            require_cls=False,
+        )
+
+    def tostr(self):
+        """
+        Returns the line marker as string. Note that fparser accepts
+        spaces before the `#`, but it should remove the spaces, hence
+        we lstrip the result
+        :return: this linemarker as a string.
+        :rtype: str
+        """
+        return self.items[0].lstrip()
+
+
 class Cpp_Error_Stmt(WORDClsBase):  # 6.10.5 Error directive
     """
     C99 6.10.5 Error directive

src/fparser/two/tests/test_c99preprocessor.py

@@ -57,6 +57,7 @@
     Cpp_Macro_Identifier_List,
     Cpp_Undef_Stmt,
     Cpp_Line_Stmt,
+    Cpp_Linemarker_Stmt,
     Cpp_Error_Stmt,
     Cpp_Warning_Stmt,
     Cpp_Null_Stmt,
@@ -451,6 +452,30 @@ def test_incorrect_line_stmt(line):
     assert "Cpp_Line_Stmt: '{0}'".format(line) in str(excinfo.value)
 
 
+@pytest.mark.usefixtures("f2003_create")
+@pytest.mark.parametrize(
+    "line, ref",
+    [
+        ('# 123 "file"', '# 123 "file"'),
+        ('  #     123 "file"', '#     123 "file"'),
+        ('# 123 "file" 1 3', '# 123 "file" 1 3'),
+    ],
+)
+def test_linemarker(line, ref):
+    """Test that #line is recognized"""
+    result = Cpp_Linemarker_Stmt(line)
+    assert str(result) == ref
+
+
+@pytest.mark.usefixtures("f2003_create")
+@pytest.mark.parametrize("line", ["# abc", '# "bla"', "# 123 'wrong_quotes'"])
+def test_incorrect_linemarker(line):
+    """Test that incorrectly formed #line statements raise exception"""
+    with pytest.raises(NoMatchError) as excinfo:
+        _ = Cpp_Linemarker_Stmt(line)
+    assert "Cpp_Linemarker_Stmt: '{0}'".format(line) in str(excinfo.value)
+
+
 @pytest.mark.usefixtures("f2003_create")
 @pytest.mark.parametrize("line", ["#error MSG", "  #  error  MSG  "])
 def test_error_statement_with_msg(line):

src/fparser/two/utils.py

@@ -336,7 +336,12 @@ def import_now():
             Comment,
             Include_Stmt,
             add_comments_includes_directives,
+            Continue_Stmt,
+            End_Do,
+            End_Do_Stmt,
+            Label_Do_Stmt,
         )
+
         from fparser.two import C99Preprocessor
 
         DynamicImport.Else_If_Stmt = Else_If_Stmt
@@ -356,6 +361,10 @@ def import_now():
         DynamicImport.add_comments_includes_directives = (
             add_comments_includes_directives
         )
+        DynamicImport.Continue_Stmt = Continue_Stmt,
+        DynamicImport.End_Do = End_Do
+        DynamicImport.End_Do_Stmt = End_Do_Stmt
+        DynamicImport.Label_Do_Stmt = Label_Do_Stmt
 
 
 di = DynamicImport()
@@ -750,6 +759,38 @@ def match(
                     i += 1
                     continue
 
+                # The grammar contains an exponential scaling behaviour for
+                # non-blocked labelled loop statements. The parser will try
+                # to find a match for a blocked do statement, but will ignore
+                # the fact that there is a non-blocking label, e.g.:
+                #     do 10 i=1, 10
+                # 10     a(i) = 1
+                # It will try to find a `10 enddo` or `10 continue` statement,
+                # ignoring the fact that the label 10 indicates that it is not
+                # a blocked loop. Full details in ticket 499.
+                # In order to avoid that, we identify if we are looking for a
+                # labelled loop which is blocked (endcls=End_Do), and have
+                # neither an `End_Do` nor a `Continue`, which has the same
+                # label: in this case we can abort looking (which will then
+                # trigger the caller to test for the next rule, which is a
+                # non-blocked loop). This breaks the exponential behaviour
+                # in case of non-blocked loops (since the parser won't look
+                # ahead till the end of the file).
+                if (startcls is di.Label_Do_Stmt and endcls is di.End_Do and
+                        hasattr(obj, "get_end_label") and
+                        (content[start_idx].get_start_label() ==
+                            obj.get_end_label()) and
+                        not isinstance(obj, (di.End_Do_Stmt, di.Continue_Stmt))):
+                    if table_name:
+                        SYMBOL_TABLES.exit_scope()
+                    # We need to put the just read statement back:
+                    obj.restore_reader(reader)
+                    # ... and then also restore all previously read content
+                    for obj in reversed(content):
+                        obj.restore_reader(reader)
+                    # ... before we abort.
+                    return None
+
                 # We got a match for this class
                 had_match = True
                 content.append(obj)
@@ -1819,7 +1860,12 @@ def match(keyword, cls, string, colons=False, require_cls=False):
             if my_match is None:
                 return None
             line = string[len(my_match.group()) :]
-            pattern_value = keyword.value
+            # If no constant return value is defined,
+            # return the matched string
+            if keyword.value:
+                pattern_value = keyword.value
+            else:
+                pattern_value = my_match.group()
 
         if not line:
             if require_cls: