Ignore files listed in .gitignore, --exclude and --extend-exclude (#173)

* update test readme

* ignore files listed in gitignore, exclude and extend-exclude

* install depends

* always compile exclude

* add ignored files

Author: bhirsz

.github/workflows/unit-tests.yml

@@ -26,7 +26,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install pytest pylama pylama_pylint coverage click robotframework==4.0
+        pip install pytest pylama pylama_pylint coverage click robotframework==4.0 pathspec>=0.8.0
 #    - name: Code quality with pylama
     - name: Run unit tests with coverage
       run:

CHANGES.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 ### Features
+- Ignore paths from .gitignore and ``--exclude`` option, allow to ignore paths through using ``--extend-exclude`` [#110](https://github.com/MarketSquare/robotframework-tidy/issues/110)
 - Add extra indent for arguments in Suite Setup/Teardown, Test Setup/Teardown in AlignSettingsSection [#155](https://github.com/MarketSquare/robotframework-tidy/issues/155)
 - OrderSettingsSection will now preserve order of imports. It can be configured to work as before and other settings order can be also preserved [#167](https://github.com/MarketSquare/robotframework-tidy/issues/167)
 - Allow to disable selected transformers [#170](https://github.com/MarketSquare/robotframework-tidy/issues/170)

docs/source/configuration/index.rst

@@ -10,37 +10,20 @@ Configuration
    config_file
    configuring_transformers
 
-Behaviour of robotidy can be changed through global options or by configuring specific transformer.
+Behaviour of *robotidy* can be changed through global options or by configuring specific transformer.
 To read more about configuring transformer go to :ref:`configuring-transformers`. To see how to configure robotidy
 using configuration files see :ref:`config-file`.
 
 .. rubric:: Command line options
 
-- ``--transform`` / ``-t``
-  Used to select what transformers will be run (overwrites default transformers list)
-- ``--configure`` / ``-c``
-  Used to configure transformers. See more :ref:`configuring-transformers`
-- ``--list`` / ``-l``
-  List available transformers.
-- ``--desc`` / ``-d`` TRANSFORMER_NAME
-  Show documentation for selected transformer. Pass ``all`` as transformer name to show documentation for all transformers.
-- ``--output`` / ``-o``
-  Path to output file where source will be saved (by default the source file is overwritten).
-- ``--config`` FILE
-  Path to configuration file.
-- ``--overwrite/--no-overwrite`` flag
-  Flag to determine if changes should be written back to file (default: --overwrite).
-- ``--diff`` flag
-  If this flag is set Robotidy will output diff view of each processed file.
-- ``--check`` flag
-  Don't overwrite files and just return status.
-- ``--spacecount`` / ``-s``
-  The number of spaces between cells (used by some of the transformers, for example ``NormalizeSeparators``).
-- ``--startline`` / ``-sl and ``--endline`` / ``-el``
-  Used by some of the transformers to narrow down part of the file that is transformed. Line numbers start from 1.
-- ``--verbose`` / ``-v``
-  More verbose output.
-- ``--version``
-  Print Robotidy version.
-- ``help``
-  Prints robotidy help.
+To list *robotidy* command line options run::
+
+    robotidy --help
+
+.. rubric:: Ignored paths
+
+Robotidy reads and ignores paths from ``.gitignore`` and ``--exclude``. You can overwrite default excludes by using
+``--exclude`` option. If you want to exclude additional paths on top of those from ``--exclude`` and ``.gitignore`` use
+``--extend-exclude`` with pattern::
+
+    robotidy --extend-exclude skip_me.robot|some_dir/* .

docs/source/external_transformers.rst

@@ -3,7 +3,7 @@
 External transformers
 -------------------------------
 It is possible to develop your own transformers. You can use module name (if it is installed in your env) or path to
-file with class to run external transformers with robotidy::
+file with class to run external transformers with *robotidy*::
 
     robotidy --transform MyTransformers.YourCustomTransformer src
     robotidy --transform C:\transformers\YourCustomTransformer2.py src

robotidy/api.py

@@ -4,13 +4,14 @@
 from typing import Optional
 
 from robotidy.app import Robotidy
-from robotidy.cli import find_and_read_config, TransformType
+from robotidy.cli import find_and_read_config, TransformType, validate_regex
+from robotidy.files import DEFAULT_EXCLUDES
 from robotidy.utils import GlobalFormattingConfig
 
 
 class RobotidyAPI(Robotidy):
     def __init__(self, src: str, output: Optional[str], **kwargs):
-        config = find_and_read_config([src])
+        config = find_and_read_config((src,))
         config = {
             k: str(v) if not isinstance(v, (list, dict)) else v
             for k, v in config.items()
@@ -24,10 +25,16 @@ def __init__(self, src: str, output: Optional[str], **kwargs):
             start_line=kwargs.get('startline', None) or int(config['startline']) if 'startline' in config else None,
             end_line=kwargs.get('endline', None) or int(config['endline']) if 'endline' in config else None
         )
+        exclude = config.get('exclude', None)
+        extend_exclude = config.get('extend_exclude', None)
+        exclude = validate_regex(exclude if exclude is not None else DEFAULT_EXCLUDES)
+        extend_exclude = validate_regex(extend_exclude)
         super().__init__(
             transformers=transformers,
             transformers_config=configurations,
             src=(),
+            exclude=exclude,
+            extend_exclude=extend_exclude,
             overwrite=False,
             show_diff=False,
             formatting_config=formatting_config,

robotidy/app.py

@@ -1,28 +1,29 @@
 from collections import defaultdict
 from difflib import unified_diff
 from pathlib import Path
-from typing import List, Tuple, Dict, Iterator, Iterable, Optional
+from typing import List, Tuple, Dict, Iterator, Iterable, Optional, Pattern
 
 import click
 from robot.api import get_model
 from robot.errors import DataError
 
 from robotidy.transformers import load_transformers
+from robotidy.files import get_paths
 from robotidy.utils import (
     StatementLinesCollector,
     decorate_diff_with_color,
     GlobalFormattingConfig,
     ModelWriter
 )
 
-INCLUDE_EXT = ('.robot', '.resource')
-
 
 class Robotidy:
     def __init__(self,
                  transformers: List[Tuple[str, List]],
                  transformers_config: List[Tuple[str, List]],
                  src: Tuple[str, ...],
+                 exclude: Pattern,
+                 extend_exclude: Pattern,
                  overwrite: bool,
                  show_diff: bool,
                  formatting_config: GlobalFormattingConfig,
@@ -31,7 +32,7 @@ def __init__(self,
                  output: Optional[Path],
                  force_order: bool
                  ):
-        self.sources = self.get_paths(src)
+        self.sources = get_paths(src, exclude, extend_exclude)
         self.overwrite = overwrite
         self.show_diff = show_diff
         self.check = check
@@ -89,28 +90,6 @@ def output_diff(self, path: str, old_model: StatementLinesCollector, new_model:
         colorized_output = decorate_diff_with_color(lines)
         click.echo(colorized_output.encode('ascii', 'ignore').decode('ascii'), color=True)
 
-    def get_paths(self, src: Tuple[str, ...]):
-        sources = set()
-        for s in src:
-            path = Path(s).resolve()
-            if path.is_file():
-                sources.add(path)
-            elif path.is_dir():
-                sources.update(self.iterate_dir(path.iterdir()))
-            elif s == '-':
-                sources.add(path)
-
-        return sources
-
-    def iterate_dir(self, paths: Iterable[Path]) -> Iterator[Path]:
-        for path in paths:
-            if path.is_file():
-                if path.suffix not in INCLUDE_EXT:
-                    continue
-                yield path
-            elif path.is_dir():
-                yield from self.iterate_dir(path.iterdir())
-
     @staticmethod
     def convert_configure(configure: List[Tuple[str, List]]) -> Dict[str, List]:
         config_map = defaultdict(list)

robotidy/cli.py

@@ -5,14 +5,16 @@
     List,
     Iterable,
     Optional,
-    Any
+    Any,
+    Pattern
 )
 
 import click
-import toml
+import re
 
 from robotidy.app import Robotidy
 from robotidy.transformers import load_transformers
+from robotidy.files import read_pyproject_config, find_and_read_config, DEFAULT_EXCLUDES
 from robotidy.utils import (
     GlobalFormattingConfig,
     split_args_from_name_or_path,
@@ -81,70 +83,6 @@ def convert(self, value, param, ctx):
         return name, args
 
 
-def find_project_root(srcs: Iterable[str]) -> Path:
-    """Return a directory containing .git, or robotidy.toml.
-    That directory will be a common parent of all files and directories
-    passed in `srcs`.
-    If no directory in the tree contains a marker that would specify it's the
-    project root, the root of the file system is returned.
-    """
-    if not srcs:
-        return Path("/").resolve()
-
-    path_srcs = [Path(Path.cwd(), src).resolve() for src in srcs]
-
-    # A list of lists of parents for each 'src'. 'src' is included as a
-    # "parent" of itself if it is a directory
-    src_parents = [
-        list(path.parents) + ([path] if path.is_dir() else []) for path in path_srcs
-    ]
-
-    common_base = max(
-        set.intersection(*(set(parents) for parents in src_parents)),
-        key=lambda path: path.parts,
-    )
-
-    for directory in (common_base, *common_base.parents):
-        if (directory / ".git").exists():
-            return directory
-
-        if (directory / "robotidy.toml").is_file():
-            return directory
-
-        if (directory / "pyproject.toml").is_file():
-            return directory
-
-    return directory
-
-
-def find_and_read_config(src_paths: Iterable[str]) -> Dict[str, Any]:
-    project_root = find_project_root(src_paths)
-    config_path = project_root / 'robotidy.toml'
-    if config_path.is_file():
-        return read_pyproject_config(str(config_path))
-    pyproject_path = project_root / 'pyproject.toml'
-    if pyproject_path.is_file():
-        return read_pyproject_config(str(pyproject_path))
-    return {}
-
-
-def load_toml_file(path: str) -> Dict[str, Any]:
-    try:
-        config = toml.load(path)
-        click.echo(f"Loaded configuration from {path}")
-        return config
-    except (toml.TomlDecodeError, OSError) as e:
-        raise click.FileError(
-            filename=path, hint=f"Error reading configuration file: {e}"
-        )
-
-
-def read_pyproject_config(path: str) -> Dict[str, Any]:
-    config = load_toml_file(path)
-    config = config.get("tool", {}).get("robotidy", {})
-    return {k.replace('--', '').replace('-', '_'): v for k, v in config.items()}
-
-
 def parse_opt(opt):
     while opt and opt[0] == '-':
         opt = opt[1:]
@@ -185,6 +123,21 @@ def read_config(ctx: click.Context, param: click.Parameter, value: Optional[str]
     ctx.default_map = default_map
 
 
+def validate_regex_callback(
+    ctx: click.Context,
+    param: click.Parameter,
+    value: Optional[str],
+) -> Optional[Pattern]:
+    return validate_regex(value)
+
+
+def validate_regex(value: Optional[str]) -> Optional[Pattern]:
+    try:
+        return re.compile(value) if value is not None else None
+    except re.error:
+        raise click.BadParameter("Not a valid regular expression")
+
+
 def print_description(name: str):
     transformers = load_transformers(None, {}, allow_disabled=True)
     transformer_by_names = {transformer.__class__.__name__: transformer for transformer in transformers}
@@ -242,6 +195,27 @@ def print_transformers_list():
     is_eager=True,
     metavar='[PATH(S)]'
 )
+@click.option(
+    "--exclude",
+    type=str,
+    callback=validate_regex_callback,
+    help=(
+        "A regular expression that matches files and directories that should be"
+        " excluded on recursive searches. An empty value means no paths are excluded."
+        " Use forward slashes for directories on all platforms."
+        f" [default: '{DEFAULT_EXCLUDES}']"
+    ),
+    show_default=False,
+)
+@click.option(
+    "--extend-exclude",
+    type=str,
+    callback=validate_regex_callback,
+    help=(
+        "Like --exclude, but adds additional files and directories on top of the"
+        " excluded ones. (Useful if you simply want to add to the default)"
+    ),
+)
 @click.option(
     "--config",
     type=click.Path(
@@ -363,6 +337,8 @@ def cli(
         transform: List[Tuple[str, List]],
         configure: List[Tuple[str, List]],
         src: Tuple[str, ...],
+        exclude: Optional[Pattern],
+        extend_exclude: Optional[Pattern],
         overwrite: bool,
         diff: bool,
         check: bool,
@@ -395,6 +371,9 @@ def cli(
         print("No source path provided. Run robotidy --help to see how to use robotidy")
         ctx.exit(0)
 
+    if exclude is None:
+        exclude = re.compile(DEFAULT_EXCLUDES)
+
     if config and verbose:
         click.echo(f'Loaded {config} configuration file')
 
@@ -408,6 +387,8 @@ def cli(
         transformers=transform,
         transformers_config=configure,
         src=src,
+        exclude=exclude,
+        extend_exclude=extend_exclude,
         overwrite=overwrite,
         show_diff=diff,
         formatting_config=formatting_config,