Merge branch 'main' into naan-hoist-after-inline

Author: reuterbal

loki/analyse/analyse_dataflow.py

@@ -10,7 +10,7 @@
 """
 
 from contextlib import contextmanager
-from loki.expression import FindVariables, Array, FindInlineCalls
+from loki.expression import FindVariables, Array, FindInlineCalls, ProcedureSymbol, FindTypedSymbols
 from loki.tools import as_tuple, flatten
 from loki.types import BasicType
 from loki.ir import Visitor, Transformer
@@ -232,9 +232,9 @@ def visit_CallStatement(self, o, **kwargs):
             outvals = [val for arg, val in o.arg_iter() if str(arg.type.intent).lower() in ('inout', 'out')]
             invals = [val for arg, val in o.arg_iter() if str(arg.type.intent).lower() in ('inout', 'in')]
 
+            arrays = [v for v in FindVariables().visit(outvals) if isinstance(v, Array)]
+            dims = set(v for a in arrays for v in self._symbols_from_expr(a.dimensions))
             for val in outvals:
-                arrays = [v for v in FindVariables().visit(outvals) if isinstance(v, Array)]
-                dims = set(v for a in arrays for v in FindVariables().visit(a.dimensions))
                 exprs = self._symbols_from_expr(val)
                 defines |= {e for e in exprs if not e in dims}
                 uses |= dims
@@ -269,12 +269,14 @@ def visit_Deallocation(self, o, **kwargs):
     visit_Nullify = visit_Deallocation
 
     def visit_Import(self, o, **kwargs):
-        defines = self._symbols_from_expr(o.symbols or ())
+        defines = set(s.clone(dimensions=None) for s in FindTypedSymbols().visit(o.symbols or ())
+                      if isinstance(s, ProcedureSymbol))
         return self.visit_Node(o, defines_symbols=defines, **kwargs)
 
     def visit_VariableDeclaration(self, o, **kwargs):
         defines = self._symbols_from_expr(o.symbols, condition=lambda v: v.type.initial is not None)
-        return self.visit_Node(o, defines_symbols=defines, **kwargs)
+        uses = {v for a in o.symbols if isinstance(a, Array) for v in self._symbols_from_expr(a.dimensions)}
+        return self.visit_Node(o, defines_symbols=defines, uses_symbols=uses, **kwargs)
 
 
 class DataflowAnalysisDetacher(Transformer):

loki/analyse/tests/test_analyse_dataflow.py

@@ -10,7 +10,7 @@
 from loki import (
     Subroutine, FindNodes, Assignment, Loop, Conditional, Pragma, fgen, Sourcefile,
     CallStatement, MultiConditional, MaskedStatement, ProcedureSymbol, WhileLoop,
-    Associate
+    Associate, Module
 )
 from loki.analyse import (
     dataflow_analysis_attached, read_after_write_vars, loop_carried_dependencies
@@ -281,6 +281,41 @@ def test_analyse_interface(frontend):
         assert isinstance(list(routine.spec.defines_symbols)[0], ProcedureSymbol)
         assert 'random_call' in routine.spec.defines_symbols
 
+
+@pytest.mark.parametrize('frontend', available_frontends())
+def test_analyse_imports(frontend):
+    fcode_module = """
+module some_mod
+implicit none
+real :: my_global
+contains
+subroutine random_call(v_out,v_in,v_inout)
+
+  real,intent(in)  :: v_in
+  real,intent(out)  :: v_out
+  real,intent(inout)  :: v_inout
+
+
+end subroutine random_call
+end module some_mod
+""".strip()
+
+    fcode = """
+subroutine test()
+use some_mod, only: my_global, random_call
+implicit none
+
+end subroutine test
+""".strip()
+
+    module = Module.from_source(fcode_module, frontend=frontend)
+    routine = Subroutine.from_source(fcode, frontend=frontend, definitions=module)
+
+    with dataflow_analysis_attached(routine):
+        assert len(routine.spec.defines_symbols) == 1
+        assert 'random_call' in routine.spec.defines_symbols
+
+
 @pytest.mark.parametrize('frontend', available_frontends())
 def test_analyse_enriched_call(frontend):
     fcode = """
@@ -429,26 +464,30 @@ def test_analyse_call_args_array_slicing(frontend):
 
 end subroutine random_call
 
-subroutine test(v,n)
+subroutine test(v,n,b)
 implicit none
 
   integer,intent(out) :: v(:)
   integer,intent( in) :: n
+  integer,intent( in) :: b(n)
 
   call random_call(v(n))
+  call random_call(v(b(1)))
 
 end subroutine test
     """.strip()
 
     source = Sourcefile.from_source(fcode, frontend=frontend)
     routine = source['test']
 
-    call = FindNodes(CallStatement).visit(routine.body)[0]
+    calls = FindNodes(CallStatement).visit(routine.body)
     routine.enrich(source.all_subroutines)
 
     with dataflow_analysis_attached(routine):
-        assert 'n' in call.uses_symbols
-        assert not 'n' in call.defines_symbols
+        assert 'n' in calls[0].uses_symbols
+        assert not 'n' in calls[0].defines_symbols
+        assert 'b' in calls[1].uses_symbols
+        assert not 'b' in calls[0].defines_symbols
 
 
 @pytest.mark.parametrize('frontend', available_frontends())

loki/batch/scheduler.py

@@ -503,7 +503,6 @@ def _get_definition_items(_item, sgraph_items):
 
         if transformation.creates_items:
             self._discover()
-
             self._parse_items()
 
     def callgraph(self, path, with_file_graph=False, with_legend=False):

loki/frontend/regex.py

@@ -900,8 +900,8 @@ def __init__(self):
         super().__init__(
             r'^(((?:type|class)[ \t]*\([ \t]*(?P<typename>\w+)[ \t]*\))|' # TYPE or CLASS keyword with typename
             r'^([ \t]*(?P<basic_type>(logical|real|integer|complex|character))'
-            r'(?P<param>\((kind|len)=[a-z0-9_-]+\))?[ \t]*))'
-            r'(?:[ \t]*,[ \t]*[a-z]+(?:\((.(\(.*\))?)*?\))?)*'  # Optional attributes
+            r'[ \t]*(?P<param>\([ \t]*(kind|len)[ \t]*=[ \t]*[a-z0-9_-]+[ \t]*\))?[ \t]*))'
+            r'(?:[ \t]*,[ \t]*[a-z]+(?:[ \t]*\((.(\(.*\))?)*?\))?)*'  # Optional attributes
             r'(?:[ \t]*::)?'  # Optional `::` delimiter
             r'[ \t]*'  # Some white space
             r'(?P<variables>\w+\b.*?)$',  # Variable names

loki/frontend/tests/test_regex_frontend.py

@@ -0,0 +1,62 @@
+# (C) Copyright 2018- ECMWF.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+"""
+Verify correct parsing behaviour of the REGEX frontend
+"""
+
+from loki.frontend import REGEX
+from loki.types import BasicType, DerivedType
+from loki.subroutine import Subroutine
+
+def test_declaration_whitespace_attributes():
+    """
+    Test correct behaviour with/without white space inside declaration attributes
+    (reported in #318).
+    """
+    fcode = """
+subroutine my_whitespace_declaration_routine(kdim, state_t0, paux)
+    use type_header, only: dimension_type, STATE_TYPE, aux_type, jprb
+    implicit none
+    TYPE( DIMENSION_TYPE) , INTENT (IN) :: KDIM
+    type (state_type  ) , intent ( in ) :: state_t0
+    TYPE (AUX_TYPE) , InteNT( In) :: PAUX
+    CHARACTER  ( LEN=10) :: STR
+    REAL(  KIND = JPRB  ) :: VAR
+end subroutine
+    """.strip()
+
+    routine = Subroutine.from_source(fcode, frontend=REGEX)
+
+    # Verify that variables and dtype information has been extracted correctly
+    assert routine.variables == ('kdim', 'state_t0', 'paux', 'str', 'var')
+    assert isinstance(routine.variable_map['kdim'].type.dtype, DerivedType)
+    assert routine.variable_map['kdim'].type.dtype.name.lower() == 'dimension_type'
+    assert isinstance(routine.variable_map['state_t0'].type.dtype, DerivedType)
+    assert routine.variable_map['state_t0'].type.dtype.name.lower() == 'state_type'
+    assert isinstance(routine.variable_map['paux'].type.dtype, DerivedType)
+    assert routine.variable_map['paux'].type.dtype.name.lower() == 'aux_type'
+    assert routine.variable_map['str'].type.dtype == BasicType.CHARACTER
+    assert routine.variable_map['var'].type.dtype == BasicType.REAL
+
+    routine.make_complete()
+
+    # Verify that additional type attributes are correct after full parse
+    assert routine.variables == ('kdim', 'state_t0', 'paux', 'str', 'var')
+    assert isinstance(routine.variable_map['kdim'].type.dtype, DerivedType)
+    assert routine.variable_map['kdim'].type.dtype.name.lower() == 'dimension_type'
+    assert routine.variable_map['kdim'].type.intent == 'in'
+    assert isinstance(routine.variable_map['state_t0'].type.dtype, DerivedType)
+    assert routine.variable_map['state_t0'].type.dtype.name.lower() == 'state_type'
+    assert routine.variable_map['state_t0'].type.intent == 'in'
+    assert isinstance(routine.variable_map['paux'].type.dtype, DerivedType)
+    assert routine.variable_map['paux'].type.dtype.name.lower() == 'aux_type'
+    assert routine.variable_map['paux'].type.intent == 'in'
+    assert routine.variable_map['str'].type.dtype == BasicType.CHARACTER
+    assert routine.variable_map['str'].type.length == 10
+    assert routine.variable_map['var'].type.dtype == BasicType.REAL
+    assert routine.variable_map['var'].type.kind == 'jprb'

loki/ir/nodes.py

@@ -1535,7 +1535,10 @@ def parent_type(self):
             return None
         if not self.parent:
             return BasicType.DEFERRED
-        return self.parent.symbol_attrs[self.extends].dtype.typedef
+        parent_type = self.parent.symbol_attrs.lookup(self.extends)
+        if not (parent_type and isinstance(parent_type.dtype, DerivedType)):
+            return BasicType.DEFERRED
+        return parent_type.dtype.typedef
 
     @property
     def declarations(self):

loki/tests/test_derived_types.py

@@ -1520,3 +1520,42 @@ def test_derived_type_symbol_inheritance(frontend):
     assert not extended_extended_type.imported_symbol_map
     #check for non-empty declarations
     assert all(decl.symbols for decl in extended_extended_type.declarations)
+
+
+@pytest.mark.parametrize('frontend', available_frontends())
+@pytest.mark.parametrize('qualified_import', (True, False))
+def test_derived_type_inheritance_missing_parent(frontend, qualified_import, tmp_path):
+    fcode_parent = """
+module parent_mod
+    implicit none
+    type, abstract, public :: parent_type
+        integer :: val
+    end type parent_type
+end module parent_mod
+    """.strip()
+
+    fcode_derived = f"""
+module derived_mod
+    use parent_mod{", only: parent_type" if qualified_import else ""}
+    implicit none
+    type, public, extends(parent_type) :: derived_type
+        integer :: val2
+    end type derived_type
+contains
+    subroutine do_something(this)
+        class(derived_type), intent(inout) :: this
+        this%val = 1
+        this%val2 = 2
+    end subroutine do_something
+end module derived_mod
+    """.strip()
+
+    parent = Module.from_source(fcode_parent, frontend=frontend, xmods=[tmp_path])
+
+    # Without enrichment we obtain only DEFERRED type information (but don't fail!)
+    derived = Module.from_source(fcode_derived, frontend=frontend, xmods=[tmp_path])
+    assert derived['derived_type'].parent_type == BasicType.DEFERRED
+
+    # With enrichment we obtain the parent type from the parent module
+    derived = Module.from_source(fcode_derived, frontend=frontend, xmods=[tmp_path], definitions=[parent])
+    assert derived['derived_type'].parent_type is parent['parent_type']

loki/transformations/data_offload.py

@@ -296,6 +296,10 @@ def transform_subroutine(self, routine, **kwargs):
                 var for var in routine.body.uses_symbols
                 if var.name in import_map or (var.parent and var.parents[0].name in import_map)
             }
+            uses_imported_symbols |= {
+                var for var in routine.spec.uses_symbols
+                if var.name in import_map or (var.parent and var.parents[0].name in import_map)
+            }
             defines_imported_symbols = {
                 var for var in routine.body.defines_symbols
                 if var.name in import_map or (var.parent and var.parents[0].name in import_map)

loki/transformations/inline.py

@@ -79,6 +79,9 @@ class InlineTransformation(Transformation):
     # Ensure correct recursive inlining by traversing from the leaves
     reverse_traversal = True
 
+    # This transformation will potentially change the edges in the callgraph
+    creates_items = False
+
     def __init__(
             self, inline_constants=False, inline_elementals=True,
             inline_internals=False, inline_marked=True,
@@ -95,6 +98,8 @@ def __init__(
         self.adjust_imports = adjust_imports
         self.external_only = external_only
         self.resolve_sequence_association = resolve_sequence_association
+        if self.inline_marked:
+            self.creates_items = True
 
     def transform_subroutine(self, routine, **kwargs):
 
@@ -211,9 +216,9 @@ def resolve_sequence_association_for_inlined_calls(routine, inline_internals, in
                     # asked sequence assoc to happen with inlining, so source for routine should be
                     # found in calls to be inlined.
                     raise ValueError(
-                        f"Cannot resolve sequence association for call to `{call.name}` " +
-                        f"to be inlined in routine `{routine.name}`, because " +
-                        f"the `CallStatement` referring to `{call.name}` does not contain " +
+                        f"Cannot resolve sequence association for call to ``{call.name}`` " +
+                        f"to be inlined in routine ``{routine.name}``, because " +
+                        f"the ``CallStatement`` referring to ``{call.name}`` does not contain " +
                         "the source code of the procedure. " +
                         "If running in batch processing mode, please recheck Scheduler configuration."
                     )

loki/transformations/tests/test_data_offload.py

@@ -256,6 +256,8 @@ def fixture_global_variable_analysis_code(tmp_path):
     integer, parameter :: nval = 5
     integer, parameter :: nfld = 3
 
+    integer :: n
+
     integer :: iarr(nfld)
     real :: rarr(nval, nfld)
 end module global_var_analysis_header_mod
@@ -297,10 +299,11 @@ def fixture_global_variable_analysis_code(tmp_path):
 
 contains
     subroutine kernel_a(arg, tt)
-        use global_var_analysis_header_mod, only: iarr, nval, nfld
+        use global_var_analysis_header_mod, only: iarr, nval, nfld, n
 
         real, intent(inout) :: arg(:,:)
         type(some_type), intent(in) :: tt
+        real :: tmp(n)
         integer :: i, j
 
         do i=1,nfld
@@ -390,7 +393,7 @@ def test_global_variable_analysis(frontend, key, config, global_variable_analysi
 
     expected_trafo_data = {
         'global_var_analysis_header_mod': {
-            'declares': {f'iarr({nfld_dim})', f'rarr({nval_dim}, {nfld_dim})'},
+            'declares': {f'iarr({nfld_dim})', f'rarr({nval_dim}, {nfld_dim})', 'n'},
             'offload': {}
         },
         'global_var_analysis_data_mod': {
@@ -402,6 +405,7 @@ def test_global_variable_analysis(frontend, key, config, global_variable_analysi
             'defines_symbols': set(),
             'uses_symbols': nval_data | nfld_data | {
                 (f'iarr({nfld_dim})', 'global_var_analysis_header_mod'),
+                ('n', 'global_var_analysis_header_mod'),
                 (f'rarr({nval_dim}, {nfld_dim})', 'global_var_analysis_header_mod')
             }
         },
@@ -416,6 +420,7 @@ def test_global_variable_analysis(frontend, key, config, global_variable_analysi
             'defines_symbols': {('rdata(:, :, :)', 'global_var_analysis_data_mod')},
             'uses_symbols': nval_data | nfld_data | {
                 ('rdata(:, :, :)', 'global_var_analysis_data_mod'),
+                ('n', 'global_var_analysis_header_mod'),
                 ('tt', 'global_var_analysis_data_mod'), ('tt%vals', 'global_var_analysis_data_mod'),
                 (f'iarr({nfld_dim})', 'global_var_analysis_header_mod'),
                 (f'rarr({nval_dim}, {nfld_dim})', 'global_var_analysis_header_mod')
@@ -465,8 +470,8 @@ def test_global_variable_offload(frontend, key, config, global_variable_analysis
 
     expected_trafo_data = {
         'global_var_analysis_header_mod': {
-            'declares': {f'iarr({nfld_dim})', f'rarr({nval_dim}, {nfld_dim})'},
-            'offload': {f'iarr({nfld_dim})', f'rarr({nval_dim}, {nfld_dim})'}
+            'declares': {f'iarr({nfld_dim})', f'rarr({nval_dim}, {nfld_dim})', 'n'},
+            'offload': {f'iarr({nfld_dim})', f'rarr({nval_dim}, {nfld_dim})', 'n'}
         },
         'global_var_analysis_data_mod': {
             'declares': {'rdata(:, :, :)', 'tt'},
@@ -486,7 +491,7 @@ def test_global_variable_offload(frontend, key, config, global_variable_analysis
 
     # Verify imports have been added to the driver
     expected_imports = {
-        'global_var_analysis_header_mod': {'iarr', 'rarr'},
+        'global_var_analysis_header_mod': {'iarr', 'rarr', 'n'},
         'global_var_analysis_data_mod': {'rdata'}
     }
 
@@ -495,7 +500,7 @@ def test_global_variable_offload(frontend, key, config, global_variable_analysis
         assert {var.name.lower() for var in import_.symbols} == expected_imports[import_.module.lower()]
 
     expected_h2d_pragmas = {
-        'update device': {'iarr', 'rdata', 'rarr'},
+        'update device': {'iarr', 'rdata', 'rarr', 'n'},
         'enter data copyin': {'tt%vals'}
     }
     expected_d2h_pragmas = {
@@ -515,7 +520,7 @@ def test_global_variable_offload(frontend, key, config, global_variable_analysis
 
     # Verify declarations have been added to the header modules
     expected_declarations = {
-        'global_var_analysis_header_mod': {'iarr', 'rarr'},
+        'global_var_analysis_header_mod': {'iarr', 'rarr', 'n'},
         'global_var_analysis_data_mod': {'rdata', 'tt'}
     }