Ignore comments in formatted Run Keywords (#707)

Author: bhirsz

docs/releasenotes/unreleased/transformers.1.rst

@@ -0,0 +1,16 @@
+Ignore comments in IndentNestedKeywords transformer (#702)
+----------------------------------------------------------
+
+``IndentNestedKeywords`` moves comments before transformation. This is required in order to properly format different
+types of the source code (especially when expanding single line to multiple lines). However as side affect
+``IndentNestedKeywords`` moved the comments even if the code didn't require formatting::
+
+    *** Test Cases ***
+    Keyword with commented out single line
+        Run Keywords
+        ...    No Operation
+        # ...    No Operation
+        ...    No Operation
+
+In such case the code is already formatted and does not require moving the comments. After this release such
+comments will be left alone in a case where the code is already formatted.

robotidy/transformers/IndentNestedKeywords.py

@@ -90,6 +90,34 @@ def parse_keyword_lines(self, lines, tokens, new_line, eol):
         tokens.append(eol)
         return tokens
 
+    @staticmethod
+    def node_was_transformed(old_tokens, new_tokens) -> bool:
+        """
+        Compare code before and after transformation while ignoring comments to check if code was transformed.
+        """
+        if len(new_tokens) > len(old_tokens):
+            return True
+        old_tokens_no_comm = []
+        data_in_line = False
+        for token in old_tokens:
+            if token.type == Token.EOL:
+                if not data_in_line:
+                    continue
+                data_in_line = False
+            elif token.type == Token.COMMENT:
+                if old_tokens_no_comm and old_tokens_no_comm[-1].type == Token.SEPARATOR:
+                    old_tokens_no_comm.pop()
+                continue
+            elif token.type != Token.SEPARATOR:
+                data_in_line = True
+            old_tokens_no_comm.append(token)
+        if len(new_tokens) != len(old_tokens_no_comm):
+            return True
+        for new_token, old_token in zip(new_tokens, old_tokens_no_comm):
+            if new_token.type != old_token.type or new_token.value != old_token.value:
+                return True
+        return False
+
     @skip_if_disabled
     def visit_SuiteSetup(self, node):  # noqa
         lines = self.get_setting_lines(node, 0)
@@ -99,8 +127,11 @@ def visit_SuiteSetup(self, node):  # noqa
         separator = self.get_separator()
         new_line = misc.get_new_line()
         tokens = [node.data_tokens[0], separator, *misc.join_tokens_with_token(lines[0][1], separator)]
-        node.tokens = self.parse_keyword_lines(lines, tokens, new_line, eol=node.tokens[-1])
-        return (*comments, node)
+        formatted_tokens = self.parse_keyword_lines(lines, tokens, new_line, eol=node.tokens[-1])
+        if self.node_was_transformed(node.tokens, formatted_tokens):
+            node.tokens = formatted_tokens
+            return (*comments, node)
+        return node
 
     visit_SuiteTeardown = visit_TestSetup = visit_TestTeardown = visit_SuiteSetup
 
@@ -146,8 +177,11 @@ def visit_KeywordCall(self, node):  # noqa
             tokens.extend([*misc.join_tokens_with_token(assign, separator), separator])
         tokens.extend(misc.join_tokens_with_token(lines[0][1], separator))
         new_line = misc.get_new_line(indent)
-        node.tokens = self.parse_keyword_lines(lines, tokens, new_line, eol=node.tokens[-1])
-        return (*comments, node)
+        formatted_tokens = self.parse_keyword_lines(lines, tokens, new_line, eol=node.tokens[-1])
+        if self.node_was_transformed(node.tokens, formatted_tokens):
+            node.tokens = formatted_tokens
+            return (*comments, node)
+        return node
 
     def split_too_long_lines(self, lines, indent):
         """

robotidy/transformers/OrderSettingsSection.py

@@ -16,7 +16,7 @@ class OrderSettingsSection(Transformer):
       - documentation (Documentation, Metadata),
       - imports (Library, Resource, Variables),
       - settings (Suite Setup and Teardown, Test Setup and Teardown, Test Timeout, Test Template),
-      - tags (Force Tags, Default Tags)
+      - tags (Force Tags, Default Tags, Test Tags)
 
     Then ordered by groups (according to ``group_order = documentation,imports,settings,tags`` order). Every
     group is separated by ``new_lines_between_groups = 1`` new lines.
@@ -216,8 +216,8 @@ def sort_builtin_libs(statements):
             if (
                 isinstance(statement, LibraryImport)
                 and statement.name
-                and statement.name != "Remote"
                 and statement.name in STDLIBS
+                and statement.name != "Remote"
             ):
                 before.append((comments, statement))
             else:

robotidy/utils/misc.py

@@ -348,10 +348,9 @@ def get_comments(tokens):
             if token.value.startswith("#"):
                 comments.append(token)
             elif comments:
-                comments[-1].value += prev_sep + token.value
+                comments[-1] = Token(Token.COMMENT, comments[-1].value + prev_sep + token.value)
             else:
-                token.value = f"# {token.value}"
-                comments.append(token)
+                comments.append(Token(Token.COMMENT, f"# {token.value}"))
         elif token.type == Token.SEPARATOR:
             prev_sep = token.value
     return comments

tests/atest/transformers/IndentNestedKeywords/expected/comments.robot

@@ -1,3 +1,16 @@
+*** Settings ***
+Suite Setup    Run Keywords
+...    No Operation
+# ...    No Operation
+# comment
+Suite Teardown    Run Keywords
+...    No Operation
+...    No Operation
+Test Setup    Run Keywords
+# ...    No Operation
+...    No Operation
+
+
 *** Keywords ***
 Comments
     # comment1  comment2
@@ -11,3 +24,18 @@ Comments
     ...        Keyword    ${arg}
     ...    ELSE
     ...        Keyword
+
+Golden Keywords
+    [Test Setup]    Run Keywords
+    ...    No Operation
+    #    No Operation
+    ...    No Operation
+
+    Run Keywords
+    ...    No Operation
+    # ...    No Operation
+    ...    No Operation
+
+    Run Keywords
+    ...    No Operation
+    ...    No Operation  # comment about this keyword call

tests/atest/transformers/IndentNestedKeywords/source/comments.robot

@@ -1,3 +1,13 @@
+*** Settings ***
+Suite Setup    Run Keywords
+...    No Operation
+# ...    No Operation
+Suite Teardown    Run Keywords    No Operation    No Operation   # comment
+Test Setup    Run Keywords
+# ...    No Operation
+...    No Operation
+
+
 *** Keywords ***
 Comments
     Run Keyword  # comment1  comment2
@@ -6,3 +16,18 @@ Comments
     ...        ${arg}  # comment 5
     ...   ELSE  # comment 6
     ...      Keyword  # comment 7
+
+Golden Keywords
+    [Test Setup]    Run Keywords
+    ...    No Operation
+    #    No Operation
+    ...    No Operation
+
+    Run Keywords
+    ...    No Operation
+    # ...    No Operation
+    ...    No Operation
+
+    Run Keywords
+    ...    No Operation
+    ...    No Operation  # comment about this keyword call