Add NormalizeTags transformer (#213)

adrszad · adrianszadkowski · bhirsz · web-flow · commit 9d44048cd54c · 2021-09-28T20:22:25.000+02:00
* Add NormalizeTags transformer

* fix line endings

* Resolve PR comments, add option to only remove duplicates

* rename parameter, resolve PR comments

* Use classes, handle comments

Co-authored-by: Adrian Szadkowski &lt;adrian.szadkowski@zebra.com&gt;
Co-authored-by: Bartłomiej Hirsz &lt;bartek.hirsz@gmail.com&gt;
diff --git a/docs/source/transformers/NormalizeTags.rst b/docs/source/transformers/NormalizeTags.rst
@@ -0,0 +1,24 @@
+.. _NormalizeTags:
+
+NormalizeTags
+================================
+
+NormalizeTags is not included in default transformers, that's why you need to call it with ``--transform`` explicitly::
+
+    robotidy --transform NormalizeTags src
+
+Or configure `enable` parameter::
+
+    robotidy --configure NormalizeTags:enabled=True
+
+
+Supported cases: lowercase (default), uppercase, titlecase.
+You can configure case using `case` parameter::
+
+    robotidy --transform NormalizeTags:case=uppercase
+
+
+You can remove duplicates without normalizing case by setting normalize_case parameter to False::
+
+    robotidy --transform NormalizeTags:normalize_case=False
+
diff --git a/robotidy/transformers/NormalizeTags.py b/robotidy/transformers/NormalizeTags.py
@@ -0,0 +1,68 @@
+from robot.api.parsing import ModelTransformer, Tags, Token, DefaultTags, ForceTags
+import click
+
+class NormalizeTags(ModelTransformer):
+    """
+    Normalize tag names by normalizing case and removing duplicates.
+    example usage:
+
+        robotidy --transform NormalizeTags:case=lowercase test.robot
+
+    Other supported cases: uppercase, titlecase. The default is lowercase.
+
+    You can also run it to remove duplicates but preserve current case by setting ``normalize_case`` parameter to False:
+
+        robotidy --transform NormalizeTags:normalize_case=False test.robot
+
+    See https://robotidy.readthedocs.io/en/latest/transformers/NormalizeTags.html for more examples.
+    """
+    CASE_FUNCTIONS = {'lowercase': str.lower, 'uppercase': str.upper, 'titlecase': str.title}
+
+    def __init__(self, case: str = 'lowercase', normalize_case: bool = True):
+        self.case = case.lower()
+        self.normalize_case = normalize_case
+        try:
+            self.case_function = self.CASE_FUNCTIONS[self.case]
+        except KeyError:
+            raise click.BadOptionUsage(
+                option_name='transform',
+                message=f"Invalid configurable value: '{case}' for case for NormalizeTags transformer. "
+                        f"Supported cases: lowercase, uppercase, titlecase.\n")
+
+    def visit_Tags(self, node):
+        return self.normalize_tags(node, Tags, indent=True)
+
+    def visit_DefaultTags(self, node):
+        return self.normalize_tags(node, DefaultTags)
+
+    def visit_ForceTags(self, node):
+        return self.normalize_tags(node, ForceTags)
+
+    def normalize_tags(self, node, tag_class, indent=False):
+        tags = [tag.value for tag in node.data_tokens[1:]]
+        if self.normalize_case:
+            tags = self.convert_case(tags)
+        tags = self.remove_duplicates(tags)
+        comments = node.get_tokens(Token.COMMENT)
+        if indent:
+            tag_node = tag_class.from_params(tags, indent=self.formatting_config.separator,
+                                             separator=self.formatting_config.separator)
+        else:
+            tag_node = tag_class.from_params(tags, separator=self.formatting_config.separator)
+        if comments:
+            tag_node.tokens = tag_node.tokens[:-1] + tuple(self.join_tokens(comments)) + (tag_node.tokens[-1],)
+        return tag_node
+
+    def convert_case(self, tags):
+        return [self.case_function(item) for item in tags]
+
+    def remove_duplicates(self, tags):
+        return list(dict.fromkeys(tags))
+
+    def join_tokens(self, tokens):
+        joined_tokens = []
+        for token in tokens:
+            joined_tokens.append(Token(Token.SEPARATOR, self.formatting_config.separator))
+            joined_tokens.append(token)
+        return joined_tokens
+
diff --git a/robotidy/transformers/__init__.py b/robotidy/transformers/__init__.py
@@ -24,6 +24,7 @@
     'NormalizeAssignments',
     'OrderSettings',
     'OrderSettingsSection',
+    'NormalizeTags',
     'OrderTags',
     'AlignSettingsSection',
     'AlignVariablesSection',
diff --git a/tests/atest/transformers/NormalizeTags/__init__.py b/tests/atest/transformers/NormalizeTags/__init__.py
diff --git a/tests/atest/transformers/NormalizeTags/expected/duplicates.robot b/tests/atest/transformers/NormalizeTags/expected/duplicates.robot
@@ -0,0 +1,44 @@
+*** Settings ***
+Documentation       OrderTags acceptance tests
+Force Tags    Tag1    tag1    tag2    tag3    tag4
+Default Tags    Tag1    tag1    tag2    tag3    tag4
+
+*** Test Cases ***
+No tags
+    Keyword no tags
+
+Different cases
+    [Tags]    Tag1    tag1    TAG1    other_tag
+    Keyword different cases
+
+Same cases
+    [Tags]    tag2    other_tag
+    Keyword same cases
+
+Not duplicates
+    [Tags]    tag3    tag4
+    Keyword not duplicates
+
+One Tag
+    [Tags]    tag5
+    One Tag Keyword
+
+*** Keywords ***
+Keyword no tags
+    No Operation
+
+Keyword same cases
+    [Tags]    tag2    other_tag
+    No Operation
+
+Keyword different cases
+    [Tags]    Tag1    tag1    TAG1    other_tag
+    No Operation
+
+Keyword not duplicates
+    [Tags]    tag3    tag4
+    No Operation
+
+One Tag Keyword
+    [Tags]    tag5
+    No Operation
diff --git a/tests/atest/transformers/NormalizeTags/expected/lowercase.robot b/tests/atest/transformers/NormalizeTags/expected/lowercase.robot
@@ -0,0 +1,47 @@
+*** Settings ***
+Documentation       OrderTags acceptance tests
+Force Tags    tag1    tag2    tag3    tag4    tag5    # comment
+Default Tags    tag1    tag2    tag3    tag4    tag5    # comment
+
+*** Test Cases ***
+No tags
+    Keyword no tags
+
+Normalize case
+    [Tags]    foo_bar_baz_a    foo_bar_baz_b    foo_bar_baz_c    foobarbazd    foobarbaze    foobarbazf    foo bar baz g    foo bar baz h    foo bar baz i
+    Keyword normalize case
+
+Deduplicate
+    [Tags]    tag_a    tag_b    tag_c
+    Keyword deduplicate
+
+Deduplicate and normalize case
+    [Tags]    foo_bar_baz_a    foo_bar_baz_b    foo_bar_baz_c    foobarbazd    foobarbaze    foobarbazf    foo bar baz g    foo bar baz h    foo bar baz i
+    Keyword deduplicate and normalize case
+
+One Tag
+    [Tags]    one_tag_various cases
+    One Tag Keyword
+
+Multiline tags
+    [Tags]    tag1    tag2    tag3    tag4    tag5    # comment1    # comment2
+
+*** Keywords ***
+Keyword no tags
+    No Operation
+
+Keyword normalize case
+    [Tags]    foo_bar_baz_a    foo_bar_baz_b    foo_bar_baz_c    foobarbazd    foobarbaze    foobarbazf    foo bar baz g    foo bar baz h    foo bar baz i
+    No Operation
+
+Keyword deduplicate
+    [Tags]    tag_a    tag_b    tag_c    # comment
+    No Operation
+
+Keyword deduplicate and normalize case
+    [Tags]    foo_bar_baz_a    foo_bar_baz_b    foo_bar_baz_c    foobarbazd    foobarbaze    foobarbazf    foo bar baz g    foo bar baz h    foo bar baz i
+    No Operation
+
+One Tag Keyword
+    [Tags]    one_tag_various cases
+    No Operation
diff --git a/tests/atest/transformers/NormalizeTags/expected/titlecase.robot b/tests/atest/transformers/NormalizeTags/expected/titlecase.robot
@@ -0,0 +1,47 @@
+*** Settings ***
+Documentation       OrderTags acceptance tests
+Force Tags    Tag1    Tag2    Tag3    Tag4    Tag5    # comment
+Default Tags    Tag1    Tag2    Tag3    Tag4    Tag5    # comment
+
+*** Test Cases ***
+No tags
+    Keyword no tags
+
+Normalize case
+    [Tags]    Foo_Bar_Baz_A    Foo_Bar_Baz_B    Foo_Bar_Baz_C    Foobarbazd    Foobarbaze    Foobarbazf    Foo Bar Baz G    Foo Bar Baz H    Foo Bar Baz I
+    Keyword normalize case
+
+Deduplicate
+    [Tags]    Tag_A    Tag_B    Tag_C
+    Keyword deduplicate
+
+Deduplicate and normalize case
+    [Tags]    Foo_Bar_Baz_A    Foo_Bar_Baz_B    Foo_Bar_Baz_C    Foobarbazd    Foobarbaze    Foobarbazf    Foo Bar Baz G    Foo Bar Baz H    Foo Bar Baz I
+    Keyword deduplicate and normalize case
+
+One Tag
+    [Tags]    One_Tag_Various Cases
+    One Tag Keyword
+
+Multiline tags
+    [Tags]    Tag1    Tag2    Tag3    Tag4    Tag5    # comment1    # comment2
+
+*** Keywords ***
+Keyword no tags
+    No Operation
+
+Keyword normalize case
+    [Tags]    Foo_Bar_Baz_A    Foo_Bar_Baz_B    Foo_Bar_Baz_C    Foobarbazd    Foobarbaze    Foobarbazf    Foo Bar Baz G    Foo Bar Baz H    Foo Bar Baz I
+    No Operation
+
+Keyword deduplicate
+    [Tags]    Tag_A    Tag_B    Tag_C    # comment
+    No Operation
+
+Keyword deduplicate and normalize case
+    [Tags]    Foo_Bar_Baz_A    Foo_Bar_Baz_B    Foo_Bar_Baz_C    Foobarbazd    Foobarbaze    Foobarbazf    Foo Bar Baz G    Foo Bar Baz H    Foo Bar Baz I
+    No Operation
+
+One Tag Keyword
+    [Tags]    One_Tag_Various Cases
+    No Operation
diff --git a/tests/atest/transformers/NormalizeTags/expected/uppercase.robot b/tests/atest/transformers/NormalizeTags/expected/uppercase.robot
@@ -0,0 +1,47 @@
+*** Settings ***
+Documentation       OrderTags acceptance tests
+Force Tags    TAG1    TAG2    TAG3    TAG4    TAG5    # comment
+Default Tags    TAG1    TAG2    TAG3    TAG4    TAG5    # comment
+
+*** Test Cases ***
+No tags
+    Keyword no tags
+
+Normalize case
+    [Tags]    FOO_BAR_BAZ_A    FOO_BAR_BAZ_B    FOO_BAR_BAZ_C    FOOBARBAZD    FOOBARBAZE    FOOBARBAZF    FOO BAR BAZ G    FOO BAR BAZ H    FOO BAR BAZ I
+    Keyword normalize case
+
+Deduplicate
+    [Tags]    TAG_A    TAG_B    TAG_C
+    Keyword deduplicate
+
+Deduplicate and normalize case
+    [Tags]    FOO_BAR_BAZ_A    FOO_BAR_BAZ_B    FOO_BAR_BAZ_C    FOOBARBAZD    FOOBARBAZE    FOOBARBAZF    FOO BAR BAZ G    FOO BAR BAZ H    FOO BAR BAZ I
+    Keyword deduplicate and normalize case
+
+One Tag
+    [Tags]    ONE_TAG_VARIOUS CASES
+    One Tag Keyword
+
+Multiline tags
+    [Tags]    TAG1    TAG2    TAG3    TAG4    TAG5    # comment1    # comment2
+
+*** Keywords ***
+Keyword no tags
+    No Operation
+
+Keyword normalize case
+    [Tags]    FOO_BAR_BAZ_A    FOO_BAR_BAZ_B    FOO_BAR_BAZ_C    FOOBARBAZD    FOOBARBAZE    FOOBARBAZF    FOO BAR BAZ G    FOO BAR BAZ H    FOO BAR BAZ I
+    No Operation
+
+Keyword deduplicate
+    [Tags]    TAG_A    TAG_B    TAG_C    # comment
+    No Operation
+
+Keyword deduplicate and normalize case
+    [Tags]    FOO_BAR_BAZ_A    FOO_BAR_BAZ_B    FOO_BAR_BAZ_C    FOOBARBAZD    FOOBARBAZE    FOOBARBAZF    FOO BAR BAZ G    FOO BAR BAZ H    FOO BAR BAZ I
+    No Operation
+
+One Tag Keyword
+    [Tags]    ONE_TAG_VARIOUS CASES
+    No Operation
diff --git a/tests/atest/transformers/NormalizeTags/source/duplicates.robot b/tests/atest/transformers/NormalizeTags/source/duplicates.robot
@@ -0,0 +1,44 @@
+*** Settings ***
+Documentation       OrderTags acceptance tests
+Force Tags    Tag1    tag1    tag2    tag2    tag3    tag4
+Default Tags    Tag1    tag1    tag2    tag2    tag3    tag4
+
+*** Test Cases ***
+No tags
+    Keyword no tags
+
+Different cases
+    [Tags]    Tag1    tag1    TAG1    other_tag
+    Keyword different cases
+
+Same cases
+    [Tags]    tag2    tag2    other_tag
+    Keyword same cases
+
+Not duplicates
+    [Tags]    tag3    tag4
+    Keyword not duplicates
+
+One Tag
+    [Tags]    tag5
+    One Tag Keyword
+
+*** Keywords ***
+Keyword no tags
+    No Operation
+
+Keyword same cases
+    [Tags]    tag2    tag2    other_tag
+    No Operation
+
+Keyword different cases
+    [Tags]    Tag1    tag1    TAG1    other_tag
+    No Operation
+
+Keyword not duplicates
+    [Tags]    tag3    tag4
+    No Operation
+
+One Tag Keyword
+    [Tags]    tag5
+    No Operation
diff --git a/tests/atest/transformers/NormalizeTags/source/tests.robot b/tests/atest/transformers/NormalizeTags/source/tests.robot
@@ -0,0 +1,53 @@
+*** Settings ***
+Documentation       OrderTags acceptance tests
+Force Tags          tag1    Tag2    TAG3    tag4    tag4    tag5    TAG5    # comment
+Default Tags        tag1    Tag2    TAG3    tag4    tag4    tag5    TAG5    # comment
+
+*** Test Cases ***
+No tags
+    Keyword no tags
+
+Normalize case
+    [Tags]    foo_bar_baz_a    Foo_Bar_Baz_b    FOO_BAR_BAZ_c     foobarbazd    FooBarBaze    FOOBARBAZf    foo bar baz g    Foo Bar Baz h    FOO BAR BAZ i
+    Keyword normalize case
+
+Deduplicate
+    [Tags]    tag_a   tag_a   tag_b    tag_c
+    Keyword deduplicate
+
+Deduplicate and normalize case
+    [Tags]    foo_bar_baz_a    foo_bar_baz_A    Foo_Bar_Baz_b    FOO_BAR_BAZ_c     foobarbazd    FooBarBaze    FOOBARBAZf    foo bar baz g    Foo Bar Baz h    FOO BAR BAZ i
+    Keyword deduplicate and normalize case
+
+One Tag
+    [Tags]    One_Tag_various CASES
+    One Tag Keyword
+
+Multiline tags
+    [Tags]    tag1
+    ...    Tag2
+    ...    TAG3    # comment1
+    ...    tag4
+    ...    tag4
+    ...    tag5    # comment2
+    ...    TAG5
+
+*** Keywords ***
+Keyword no tags
+    No Operation
+
+Keyword normalize case
+    [Tags]    foo_bar_baz_a    Foo_Bar_Baz_b    FOO_BAR_BAZ_c     foobarbazd    FooBarBaze    FOOBARBAZf    foo bar baz g    Foo Bar Baz h    FOO BAR BAZ i
+    No Operation
+
+Keyword deduplicate
+    [Tags]    tag_a   tag_a   tag_b    tag_c    # comment
+    No Operation
+
+Keyword deduplicate and normalize case
+    [Tags]    foo_bar_baz_a    foo_bar_baz_A    Foo_Bar_Baz_b    FOO_BAR_BAZ_c     foobarbazd    FooBarBaze    FOOBARBAZf    foo bar baz g    Foo Bar Baz h    FOO BAR BAZ i
+    No Operation
+
+One Tag Keyword
+    [Tags]    One_Tag_various CASES
+    No Operation
diff --git a/tests/atest/transformers/NormalizeTags/test_transformer.py b/tests/atest/transformers/NormalizeTags/test_transformer.py
