Autodetect line endings in the file (#215)

* run black

* detect and preserve line endings

Author: bhirsz

CHANGES.md

@@ -21,9 +21,11 @@
 - Non default transformers can be enabled using ``enabled=True`` parameter ([#182](https://github.com/MarketSquare/robotframework-tidy/issues/182))
 - Semicolon in parameter value can now be escaped with `\:` ([#190](https://github.com/MarketSquare/robotframework-tidy/issues/190))
 - Default separator can be changed from space to tabular with new ``--separator`` option ([#184](https://github.com/MarketSquare/robotframework-tidy/issues/184))
+- ``--lineseparator`` option now accepts `auto` value for preserving line endings found in the file ([#209](https://github.com/MarketSquare/robotframework-tidy/issues/209))
 
 ### Fixes
 - Robotidy will not print "Loaded configuration from ... " if the configuration is empty ([#193](https://github.com/MarketSquare/robotframework-tidy/issues/193))
+- no source path provided error now exits with code 1 instead of 0 ([#208](https://github.com/MarketSquare/robotframework-tidy/issues/208))
 
 ### Other
 - ReplaceRunKeywordIf now removes ELSE branch if it is unnecessary ([#192](https://github.com/MarketSquare/robotframework-tidy/issues/192))

robotidy/app.py

@@ -1,3 +1,4 @@
+import os
 import sys
 from collections import defaultdict
 from difflib import unified_diff
@@ -24,8 +25,8 @@ def __init__(
         transformers: List[Tuple[str, List]],
         transformers_config: List[Tuple[str, List]],
         src: Tuple[str, ...],
-        exclude: Pattern,
-        extend_exclude: Pattern,
+        exclude: Optional[Pattern],
+        extend_exclude: Optional[Pattern],
         overwrite: bool,
         show_diff: bool,
         formatting_config: GlobalFormattingConfig,
@@ -68,7 +69,7 @@ def transform_files(self):
                     if stdin:
                         self.print_to_stdout(new_model)
                     else:
-                        self.save_model(model)
+                        self.save_model(model.source, model)
             except DataError:
                 click.echo(
                     f"Failed to decode {source}. Default supported encoding by Robot Framework is UTF-8. Skipping file"
@@ -93,10 +94,22 @@ def print_to_stdout(self, collected_lines):
         if not self.show_diff:
             click.echo(collected_lines.text)
 
-    def save_model(self, model):
+    def save_model(self, source, model):
         if self.overwrite:
             output = self.output or model.source
-            ModelWriter(output=output, newline=self.formatting_config.line_sep).write(model)
+            ModelWriter(output=output, newline=self.get_line_ending(source)).write(model)
+
+    def get_line_ending(self, path: str):
+        if self.formatting_config.line_sep == "auto":
+            with open(path) as f:
+                f.readline()
+                if f.newlines is None:
+                    return os.linesep
+                if isinstance(f.newlines, str):
+                    return f.newlines
+                else:
+                    return f.newlines[0]
+        return self.formatting_config.line_sep
 
     def output_diff(
         self,

robotidy/cli.py

@@ -254,12 +254,13 @@ def print_transformers_list():
 @click.option(
     "-ls",
     "--lineseparator",
-    type=click.types.Choice(["native", "windows", "unix"]),
+    type=click.types.Choice(["native", "windows", "unix", "auto"]),
     default="native",
     help="Line separator to use in outputs.\n"
-    "native:  use operating system's native line separators\n"
-    "windows: use Windows line separators (CRLF)\n"
-    "unix:    use Unix line separators (LF)",
+    "native:  use operating system's native line endings\n"
+    "windows: use Windows line endings (CRLF)\n"
+    "unix:    use Unix line endings (LF)\n"
+    "auto:    maintain existing line endings (uses what's used in the first line)",
     show_default=True,
 )
 @click.option(

robotidy/files.py

@@ -92,7 +92,7 @@ def path_is_excluded(
     return bool(match and match.group(0))
 
 
-def get_paths(src: Tuple[str, ...], exclude: Pattern, extend_exclude: Optional[Pattern]):
+def get_paths(src: Tuple[str, ...], exclude: Optional[Pattern], extend_exclude: Optional[Pattern]):
     root = find_project_root(src)
     gitignore = get_gitignore(root)
     sources = set()
@@ -113,16 +113,16 @@ def get_paths(src: Tuple[str, ...], exclude: Pattern, extend_exclude: Optional[P
 
 def iterate_dir(
     paths: Iterable[Path],
-    exclude: Pattern,
-    extend_exclude: Pattern,
+    exclude: Optional[Pattern],
+    extend_exclude: Optional[Pattern],
     gitignore: Optional[PathSpec],
 ) -> Iterator[Path]:
     for path in paths:
         if gitignore is not None and gitignore.match_file(path):
             continue
         if path_is_excluded(str(path), exclude) or path_is_excluded(str(path), extend_exclude):
             continue
-        if path.is_dir() and not exclude.match(path.name):
+        if path.is_dir() and exclude and not exclude.match(path.name):
             yield from iterate_dir(
                 path.iterdir(),
                 exclude,

robotidy/utils.py

@@ -1,6 +1,6 @@
 import os
 import ast
-from typing import List
+from typing import List, Optional
 import difflib
 
 from robot.api.parsing import ModelVisitor, Token
@@ -31,8 +31,8 @@ def __init__(
         self,
         space_count: int,
         line_sep: str,
-        start_line: int,
-        end_line: int,
+        start_line: Optional[int],
+        end_line: Optional[int],
         separator: str,
     ):
         self.start_line = start_line
@@ -49,6 +49,8 @@ def __init__(
             self.line_sep = "\r\n"
         elif line_sep == "unix":
             self.line_sep = "\n"
+        elif line_sep == "auto":
+            self.line_sep = "auto"
         else:
             self.line_sep = os.linesep
 

tests/utest/test_utils.py

@@ -1,10 +1,41 @@
+import os
+from pathlib import Path
+
 import pytest
+
+from robotidy.app import Robotidy
 from robotidy.utils import (
     decorate_diff_with_color,
-    split_args_from_name_or_path
+    split_args_from_name_or_path,
+    GlobalFormattingConfig
 )
 
 
+@pytest.fixture
+def app():
+    formatting_config = GlobalFormattingConfig(
+        space_count=4,
+        line_sep="auto",
+        start_line=None,
+        separator="space",
+        end_line=None,
+    )
+    return Robotidy(
+        transformers=[],
+        transformers_config=[],
+        src=('.',),
+        exclude=None,
+        extend_exclude=None,
+        overwrite=False,
+        show_diff=False,
+        formatting_config=formatting_config,
+        verbose=False,
+        check=False,
+        output=None,
+        force_order=False
+    )
+
+
 class TestUtils:
     def test_not_changed_lines_not_colorized(self):
         lines = [
@@ -49,3 +80,27 @@ def test_split_args_from_name_or_path(self, name_or_path, expected_name, expecte
         name, args = split_args_from_name_or_path(name_or_path)
         assert name == expected_name
         assert args == expected_args
+
+    @pytest.mark.parametrize("line_sep, source_file, expected", [
+        ("auto", "lf.robot", "\n"),
+        ("auto", "crlf.robot", "\r\n"),
+        ("auto", "cr.robot", "\r"),
+        ("auto", "crlf_mixed.robot", "\n"),
+        ("auto", "empty.robot", os.linesep),
+        ("native", "lf.robot", os.linesep),
+        ("native", "crlf.robot", os.linesep),
+        ("windows", "lf.robot", "\r\n"),
+        ("windows", "crlf.robot", "\r\n"),
+        ("unix", "lf.robot", "\n"),
+        ("unix", "crlf.robot", "\n")
+    ])
+    def test_get_line_ending(self, line_sep, source_file, expected, app):
+        source = str(Path(__file__).parent / 'testdata' / 'auto_line_sep' / source_file)
+        app.formatting_config = GlobalFormattingConfig(
+            space_count=4,
+            line_sep=line_sep,
+            start_line=None,
+            separator="space",
+            end_line=None,
+        )
+        assert app.get_line_ending(source) == expected

tests/utest/testdata/auto_line_sep/.gitattributes

@@ -0,0 +1 @@
+*.robot -text

tests/utest/testdata/auto_line_sep/cr.robot

@@ -0,0 +1 @@
+*** Settings ***Library     stuff.py

tests/utest/testdata/auto_line_sep/crlf.robot

@@ -0,0 +1,2 @@
+*** Settings ***
+Library     stuff.py

tests/utest/testdata/auto_line_sep/crlf_mixed.robot

@@ -0,0 +1,2 @@
+*** Settings ***
+Library     stuff.py