POC - linter: missing kind specifier for real literals and possible fix via git patch

lint_rules/lint_rules/ifs_arpege_coding_standards.py

@@ -14,6 +14,7 @@
 
 from collections import defaultdict
 import re
+import difflib
 
 try:
     from fparser.two.Fortran2003 import Intrinsic_Name
@@ -22,15 +23,282 @@
     _intrinsic_fortran_names = ()
 
 from loki import (
-    FindInlineCalls, FindNodes, GenericRule, Module, RuleType
+    FindInlineCalls, FindNodes, GenericRule, Module, RuleType,
+    ExpressionFinder, ExpressionRetriever, FloatLiteral,
+    SubstituteExpressions
 )
-from loki import ir
-
+from loki import ir, fgen
+from loki.frontend.util import read_file
 
 __all__ = [
     'MissingImplicitNoneRule', 'OnlyParameterGlobalVarRule', 'MissingIntfbRule',
+    'MissingKindSpecifierRealLiterals'
 ]
 
+jprd_files = [] 
+
+class FindFloatLiterals(ExpressionFinder):
+    """
+    A visitor to collects :any:`FloatLiteral` used in an IR tree.
+
+    See :class:`ExpressionFinder`
+    """
+    retriever = ExpressionRetriever(lambda e: isinstance(e, (FloatLiteral,)))
+
+class MissingKindSpecifierRealLiterals(GenericRule):
+    """
+    ...
+    """
+
+    type = RuleType.SERIOUS
+    fixable = True
+
+    docs = {
+        'id': 'L0',
+        'title': (
+            'Real Literals must have a kind specifier. '
+        ),
+    }
+
+
+    @classmethod
+    def check_subroutine(cls, subroutine, rule_report, config, **kwargs):
+        """
+        ...
+        """
+        literal_nodes = FindFloatLiterals(with_ir_node=True).visit(subroutine.body)
+        for node, literals in literal_nodes:
+            for literal in literals:
+                if literal.kind is None:
+                    rule_report.add(f'Real/Float literal without kind specifier "{literal}"', node)
+
+    @classmethod
+    def fix_subroutinei_test_2(cls, subroutine, rule_report, config, sourcefile=None):
+        """
+        ...
+        """
+        for node, literals in literal_nodes:
+            literal_map = {}
+            for literal in literals:
+                if literal.kind is None and 'e' not in literal.value.lower() and 'd' not in literal.value.lower():
+                    literal_map[literal] = FloatLiteral(value=literal.value, kind='JPRB')
+            if literal_map:
+                # fixed_node = SubstituteExpressions(literal_map).visit(node)
+                # for key in literal_map:
+                #     fixed_node = re.sub(rf'{re.escape()}',
+                #             , content, flags = re.S)
+                indent = int((len(node.source.string) - len(node.source.string.lstrip(' ')))/2)
+                fixed_node_str = fgen(fixed_node, depth=indent)
+                with open (f'loki_lint_{subroutine.name}_new_file_fixed_node_str.F90', 'w') as f:
+                    f.write(fixed_node_str)
+                content_new = re.sub(rf'{re.escape(node.source.string)}',
+                        fixed_node_str, content, flags = re.S)
+                content = content_new
+        with open (f'loki_lint_{subroutine.name}_new_file.F90', 'w') as f:
+            f.write(content_new)
+        diff = difflib.unified_diff(original_content.splitlines(), content_new.splitlines(),
+                f'a/{sourcefile.path}', f'b/{sourcefile.path}', lineterm='')
+        diff_str = '\n'.join(list(diff))
+        # print(f"---{sourcefile.path}------")
+        # print(diff_str)
+        # print(f"--------------------------")
+        with open (f'loki_lint_{subroutine.name}.approach_2.patch', 'w') as f:
+            f.write(diff_str)
+            f.write('\n')
+
+    @classmethod
+    def fix_subroutine_test(cls, subroutine, rule_report, config, sourcefile=None):
+        """
+        ...
+        """
+        # sourcefile = subroutine.source.file
+        print(f"fix_subroutine: subroutine: {subroutine} | subroutine.source: {subroutine.source} | subroutine.source.file: {subroutine.source.file}")
+        original_content = read_file(str(sourcefile.path))
+        content = original_content
+        literals = FindFloatLiterals(with_ir_node=False).visit(subroutine.body)
+        literal_map = {}
+        for literal in literals:
+            if literal.kind is None:
+                literal_map[literal] = FloatLiteral(value=literal.value, kind='JPRB')
+        # content_new = content
+        if literal_map:
+            for key in literal_map:
+                # print(f"replace ")
+                # content_new = re.sub(rf'{re.escape(str(key))}', rf'{re.escape(str(literal_map[key]))}', content, flags = re.S)
+                content_new = re.sub(rf'{re.escape(str(key))}', str(literal_map[key]), content, flags = re.S)
+                content = content_new
+        diff = difflib.unified_diff(original_content.splitlines(), content_new.splitlines(),
+                f'a/{sourcefile.path}', f'b/{sourcefile.path}', lineterm='')
+        diff_str = '\n'.join(list(diff))
+        # print(f"---{sourcefile.path}------")
+        # print(diff_str)
+        # print(f"--------------------------")
+        with open (f'loki_lint_{subroutine.name}.approach_2.patch', 'w') as f:
+            f.write(diff_str)
+            f.write('\n')
+
+        """
+        for node, literals in literal_nodes:
+            literal_map = {}
+            for literal in literals:
+                if literal.kind is None:
+                    literal_map[literal] = FloatLiteral(value=literal.value, kind='JPRB')
+            if literal_map:
+                # fixed_node = SubstituteExpressions(literal_map).visit(node)
+                # indent = int((len(node.source.string) - len(node.source.string.lstrip(' ')))/2)
+                # fixed_node_str = fgen(fixed_node, depth=indent)
+                # with open (f'loki_lint_{subroutine.name}_new_file_fixed_node_str.F90', 'w') as f:
+                #     f.write(fixed_node_str)
+                # content_new = re.sub(rf'{re.escape(node.source.string)}',
+                #         fixed_node_str, content, flags = re.S)
+                # content = content_new
+        with open (f'loki_lint_{subroutine.name}_new_file.F90', 'w') as f:
+            f.write(content_new)
+        diff = difflib.unified_diff(original_content.splitlines(), content_new.splitlines(),
+                f'a/{sourcefile.path}', f'b/{sourcefile.path}', lineterm='')
+        diff_str = '\n'.join(list(diff))
+        # print(f"---{sourcefile.path}------")
+        # print(diff_str)
+        # print(f"--------------------------")
+        with open (f'loki_lint_{subroutine.name}.approach_2.patch', 'w') as f:
+            f.write(diff_str)
+            f.write('\n')
+        """
+
+    @classmethod
+    def fix_subroutine(cls, subroutine, rule_report, config, sourcefile=None):
+        """
+        ...
+        """
+        # sourcefile = subroutine.source.file
+        print(f"fix_subroutine: subroutine: {subroutine} | subroutine.source: {subroutine.source} | subroutine.source.file: {subroutine.source.file} | {str(sourcefile.path)}")
+        orig_file = str(sourcefile.path).replace('source/ifs-source/', '')
+        subdir = orig_file.split('/')[0]
+        if orig_file in jprd_files:
+            with open(f"files_to_commit_{subdir}_jprd.txt", "a") as myfile:
+                myfile.write(f"{orig_file}\n")
+            kind_spec = 'JPRD'
+        else:
+            with open(f"files_to_commit_{subdir}_jprm.txt", "a") as myfile:
+                myfile.write(f"{orig_file}\n")
+            kind_spec = 'JPRM'
+        original_content = read_file(str(sourcefile.path))
+        content = original_content
+        literal_nodes = FindFloatLiterals(with_ir_node=True).visit(subroutine.body)
+        content_new = None
+        imports = FindNodes(ir.Import).visit(subroutine.spec)
+        imp_map = {}
+        parkind1_available = False
+        substitutions = 0
+        for imp in imports:
+            if imp.module.lower() == 'parkind1':
+                parkind1_available = True
+                imp_map[imp] = imp.clone(symbols=imp.symbols + (imp.symbols[0].clone(name='JPRQ'),))
+        # print(f"imp_map: {imp_map}")
+        if not parkind1_available:
+            with open(f"files_skipped_{subdir}.txt", "a") as myfile:
+                myfile.write(f"{orig_file} since parkind1 not avail\n")
+            print(f"no parkind1 in {str(sourcefile.path)} avail, thus early exit ...")
+            return
+        for node, literals in literal_nodes:
+            # print(f"node.source: {node.source.string} | {type(node.source.string)}")
+            literal_map = {}
+            for literal in literals:
+                if literal.kind is None and 'e' not in str(literal.value).lower() and 'd' not in str(literal.value).lower():
+                    literal_map[literal] = FloatLiteral(value=literal.value, kind=kind_spec)
+            if literal_map:
+                # fixed_node = SubstituteExpressions(literal_map).visit(node)
+                fixed_node = node.source.string
+                # if hasattr(node, 'comment') and node.comment is not None:
+                #     comment = node.comment
+                #     fixed_node._update(comment=None)
+                # else:
+                #     comment = None 
+                for key in literal_map:
+                    fixed_node = re.sub(rf'{re.escape(str(key))}(?!(_{kind_spec}|_JP|[0-9]|[eEdD]))',
+                        str(literal_map[key]), fixed_node, flags = re.S)
+                # indent = int((len(node.source.string) - len(node.source.string.lstrip(' ')))/2)
+                # fixed_node_str = fgen(fixed_node, depth=indent)
+                # if comment is not None:
+                #     fixed_node._update(comment=comment)
+                fixed_node_str = str(fixed_node)
+                # with open (f'loki_lint_{subroutine.name}_new_file_fixed_node_str.F90', 'w') as f:
+                #     f.write(fixed_node_str)
+                # content_new = re.sub(rf'{re.escape(node.source.string)}(?!_JPRB)(?<!JPRB)$',
+                content_new, subs = re.subn(rf'{re.escape(node.source.string)}(?!(_{kind_spec}|_JP|[0-9]|[eEdD]))',
+                        fixed_node_str, content, flags = re.S)
+                substitutions += subs
+                content = content_new
+        # with open (f'loki_lint_{subroutine.name}_new_file.F90', 'w') as f:
+        #     f.write(content_new)
+        if content_new is not None and subs > 0:
+            _subroutine = content_new.lower().find(f'subroutine {subroutine.name.lower()}')
+            _function = content_new.lower().find(f'function {subroutine.name.lower()}')
+            if _subroutine != -1:
+                index = _subroutine
+            if _function != -1:
+                index = _function
+            if index != -1:
+                for node in imp_map:
+                    fixed_node = node.source.string
+                    # fixed_node = re.sub(rf'{re.escape(str(key))}',
+                    #         str(imp_map[node]), fixed_node, flags = re.S)
+                    # fixed_node_str = f'{str(fixed_node)}, JPRQ'
+                    fixed_node_str = str(fixed_node)
+                    if kind_spec not in fixed_node_str.upper():
+                        fixed_node_str = f'{str(fixed_node)}, {kind_spec}'
+                        print(f"fixed_node_str: {fixed_node_str}")
+                        content_new = re.sub(rf'{re.escape(node.source.string)}',
+                                fixed_node_str, content[index:], flags = re.S, count=1)
+                        # content[index::] = content_new
+                        # content = content[:index-1] + content_new
+                        content_new = content[:index] + content_new
+            diff = difflib.unified_diff(original_content.splitlines(), content_new.splitlines(),
+                    f'a/{sourcefile.path}', f'b/{sourcefile.path}', lineterm='')
+            diff_str = '\n'.join(list(diff))
+            # print(f"---{sourcefile.path}------")
+            # print(diff_str)
+            # print(f"--------------------------")
+            with open (f'loki_lint_{subroutine.name}.patch', 'w') as f:
+                f.write(diff_str)
+                f.write('\n')
+
+    @classmethod
+    def fix_subroutine_working(cls, subroutine, rule_report, config, sourcefile=None):
+        """
+        ...
+        """
+        # sourcefile = subroutine.source.file
+        print(f"fix_subroutine: subroutine: {subroutine} | subroutine.source: {subroutine.source} | subroutine.source.file: {subroutine.source.file}")
+        original_content = read_file(str(sourcefile.path))
+        content = original_content
+        literal_nodes = FindFloatLiterals(with_ir_node=True).visit(subroutine.body)
+        for node, literals in literal_nodes:
+            # print(f"node.source: {node.source.string} | {type(node.source.string)}")
+            literal_map = {}
+            for literal in literals:
+                if literal.kind is None:
+                    literal_map[literal] = FloatLiteral(value=literal.value, kind='JPRB')
+            if literal_map:
+                fixed_node = SubstituteExpressions(literal_map).visit(node)
+                indent = int((len(node.source.string) - len(node.source.string.lstrip(' ')))/2)
+                fixed_node_str = fgen(fixed_node, depth=indent)
+                with open (f'loki_lint_{subroutine.name}_new_file_fixed_node_str.F90', 'w') as f:
+                    f.write(fixed_node_str)
+                content_new = re.sub(rf'{re.escape(node.source.string)}',
+                        fixed_node_str, content, flags = re.S)
+                content = content_new
+        with open (f'loki_lint_{subroutine.name}_new_file.F90', 'w') as f:
+            f.write(content_new)
+        diff = difflib.unified_diff(original_content.splitlines(), content_new.splitlines(),
+                f'a/{sourcefile.path}', f'b/{sourcefile.path}', lineterm='')
+        diff_str = '\n'.join(list(diff))
+        # print(f"---{sourcefile.path}------")
+        # print(diff_str)
+        # print(f"--------------------------")
+        with open (f'loki_lint_{subroutine.name}.patch', 'w') as f:
+            f.write(diff_str)
+            f.write('\n')
 
 class MissingImplicitNoneRule(GenericRule):
     """

loki/expression/mappers.py

@@ -173,7 +173,7 @@ def map_quotient(self, expr, enclosing_prec, *args, **kwargs):
         numerator = self.rec_with_force_parens_around(expr.numerator, PREC_PRODUCT, *args, **kwargs)
         kwargs['force_parens_around'] = self.multiplicative_primitives
         denominator = self.rec_with_force_parens_around(expr.denominator, PREC_PRODUCT, *args, **kwargs)
-        return self.parenthesize_if_needed(self.format('%s / %s', numerator, denominator),
+        return self.parenthesize_if_needed(self.format('%s/%s', numerator, denominator),
                                            enclosing_prec, PREC_PRODUCT)
 
     def map_parenthesised_add(self, expr, enclosing_prec, *args, **kwargs):
@@ -206,7 +206,7 @@ def map_inline_do(self, expr, enclosing_prec, *args, **kwargs):
 
     def map_array_subscript(self, expr, enclosing_prec, *args, **kwargs):
         name_str = self.rec(expr.aggregate, PREC_NONE, *args, **kwargs)
-        index_str = self.join_rec(', ', expr.index_tuple, PREC_NONE, *args, **kwargs)
+        index_str = self.join_rec(',', expr.index_tuple, PREC_NONE, *args, **kwargs)
         return f'{name_str}({index_str})'
 
     map_string_subscript = map_array_subscript

loki/frontend/tests/test_frontends.py

@@ -22,7 +22,7 @@
 from loki import (
     Module, Subroutine, FindVariables, BasicType, config, Sourcefile,
     RawSource, RegexParserClass, ProcedureType, DerivedType,
-    PreprocessorDirective, config_override
+    PreprocessorDirective, config_override, FindLiterals
 )
 from loki.build import jit_compile, clean_test
 from loki.expression import symbols as sym
@@ -53,7 +53,23 @@ def fixture_reset_regex_frontend_timeout():
     yield
     config['regex-frontend-timeout'] = original_timeout
 
+@pytest.mark.parametrize('frontend', available_frontends())
+def test_literals_kind(frontend):
+    """
+    """
 
+    fcode = """
+SUBROUTINE SOME_SUBROUTINE()
+REAL :: A
+A = 0.d0
+END SUBROUTINE
+    """.strip()
+    routine = Subroutine.from_source(fcode, frontend=frontend)
+    literals = FindLiterals().visit(routine.body)
+    # print(f"literals: {literals}")
+    for literal in literals:
+        print(f"literal: '{literal}' | {literal.kind}")
+
 @pytest.mark.parametrize('frontend', available_frontends())
 def test_check_alloc_opts(here, frontend):
     """

loki/ir/pprint.py

@@ -150,7 +150,8 @@ def format_line(self, *items, comment=None, no_wrap=False, no_indent=False):
                  required to observe the line width limit.
         :rtype: str
         """
-        if not no_indent:
+        # print(f"items: {items}")
+        if not no_indent and items != ('',):
             items = [self.indent, *items]
         if no_wrap:
             # Simply concatenate items and append the comment

loki/lint/linter.py

@@ -226,7 +226,7 @@ def fix(self, sourcefile, file_report, backup_suffix=None, overwrite_config=None
         sourcefile = Fixer.fix(sourcefile, file_report.fixable_reports, config)
 
         # Create the the source string for the output
-        sourcefile.write(conservative=True)
+        # sourcefile.write(conservative=True)
 
 
 class LinterTransformation(Transformation):
@@ -291,7 +291,15 @@ def check_and_fix_file(path, linter, fix=False, backup_suffix=None):
         if fix:
             linter.fix(source, report, backup_suffix=backup_suffix)
     except Exception as exc:  # pylint: disable=broad-except
-        linter.reporter.add_file_error(path, type(exc), str(exc))
+        print(f"exc: {exc}")
+        orig_file = str(path).replace('source/ifs-source/', '')
+        subdir = orig_file.split('/')[0]
+        with open(f"files_exception_{subdir}.txt", "a") as myfile:
+            myfile.write(f"{orig_file}: {exc}\n")
+        try:
+            linter.reporter.add_file_error(path, type(exc), str(exc))
+        except Exception as e:
+            print(f" ... e: {e}")
         if loki_config['debug']:
             raise exc
         return False

loki/lint/rules.py

@@ -175,15 +175,15 @@ def check(cls, ast, rule_report, config, **kwargs):
                     cls.check(member, rule_report, config, **kwargs)
 
     @classmethod
-    def fix_module(cls, module, rule_report, config):
+    def fix_module(cls, module, rule_report, config, sourcefile=None):
         """
         Fix rule violations on module level
 
         Must be implemented by a rule if applicable.
         """
 
     @classmethod
-    def fix_subroutine(cls, subroutine, rule_report, config):
+    def fix_subroutine(cls, subroutine, rule_report, config, sourcefile=None):
         """
         Fix rule violations on subroutine level