Added --include and --exclude cli options (#281)

These 2 options allow you to pass in regular expressions that determine
whether files/directories are included or excluded in the recursive file
search.

Fixes #270

README.md

@@ -89,6 +89,17 @@ Options:
                               **kwargs.  [default: per-file auto-detection]
   -S, --skip-string-normalization
                               Don't normalize string quotes or prefixes.
+  --include TEXT              A regular expression that matches files and
+                              directories that should be included on
+                              recursive searches. On Windows, use forward
+                              slashes for directories.  [default: \.pyi?$]
+  --exclude TEXT              A regular expression that matches files and
+                              directories that should be excluded on
+                              recursive searches. On Windows, use forward
+                              slashes for directories.  [default:
+                              build/|buck-out/|dist/|_build/|\.git/|\.hg/|
+                              \.mypy_cache/|\.tox/|\.venv/]
+
   --version                   Show the version and exit.
   --help                      Show this message and exit.
 ```
@@ -698,6 +709,8 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
 
 ### 18.6b0
 
+* added `--include` and `--exclude` (#270)
+
 * added `--skip-string-normalization` (#118)
 
 * fixed stdin handling not working correctly if an old version of Click was

black.py

@@ -46,6 +46,10 @@
 
 __version__ = "18.5b1"
 DEFAULT_LINE_LENGTH = 88
+DEFAULT_EXCLUDES = (
+    r"build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.tox/|\.venv/"
+)
+DEFAULT_INCLUDES = r"\.pyi?$"
 CACHE_DIR = Path(user_cache_dir("black", version=__version__))
 
 
@@ -189,6 +193,28 @@ class FileMode(Flag):
     is_flag=True,
     help="Don't normalize string quotes or prefixes.",
 )
+@click.option(
+    "--include",
+    type=str,
+    default=DEFAULT_INCLUDES,
+    help=(
+        "A regular expression that matches files and directories that should be "
+        "included on recursive searches. On Windows, use forward slashes for "
+        "directories."
+    ),
+    show_default=True,
+)
+@click.option(
+    "--exclude",
+    type=str,
+    default=DEFAULT_EXCLUDES,
+    help=(
+        "A regular expression that matches files and directories that should be "
+        "excluded on recursive searches. On Windows, use forward slashes for "
+        "directories."
+    ),
+    show_default=True,
+)
 @click.version_option(version=__version__)
 @click.argument(
     "src",
@@ -208,14 +234,26 @@ def main(
     py36: bool,
     skip_string_normalization: bool,
     quiet: bool,
+    include: str,
+    exclude: str,
     src: List[str],
 ) -> None:
     """The uncompromising code formatter."""
     sources: List[Path] = []
+    try:
+        include_regex = re.compile(include)
+    except re.error:
+        err(f"Invalid regular expression for include given: {include!r}")
+        ctx.exit(2)
+    try:
+        exclude_regex = re.compile(exclude)
+    except re.error:
+        err(f"Invalid regular expression for exclude given: {exclude!r}")
+        ctx.exit(2)
     for s in src:
         p = Path(s)
         if p.is_dir():
-            sources.extend(gen_python_files_in_dir(p))
+            sources.extend(gen_python_files_in_dir(p, include_regex, exclude_regex))
         elif p.is_file():
             # if a file was explicitly given, we don't care about its extension
             sources.append(p)
@@ -2750,33 +2788,35 @@ def get_future_imports(node: Node) -> Set[str]:
     return imports
 
 
-PYTHON_EXTENSIONS = {".py", ".pyi"}
+def gen_python_files_in_dir(
-BLACKLISTED_DIRECTORIES = {
+    path: Path, include: Pattern[str], exclude: Pattern[str]
-    "build",
+) -> Iterator[Path]:
-    "buck-out",
+    """Generate all files under `path` whose paths are not excluded by the
-    "dist",
+    `exclude` regex, but are included by the `include` regex.
-    "_build",
-    ".git",
-    ".hg",
-    ".mypy_cache",
-    ".tox",
-    ".venv",
-}
-
-
-def gen_python_files_in_dir(path: Path) -> Iterator[Path]:
-    """Generate all files under `path` which aren't under BLACKLISTED_DIRECTORIES
-    and have one of the PYTHON_EXTENSIONS.
     """
+
     for child in path.iterdir():
+        searchable_path = str(child.as_posix())
+        if Path(child.parts[0]).is_dir():
+            searchable_path = "/" + searchable_path
         if child.is_dir():
-            if child.name in BLACKLISTED_DIRECTORIES:
+            searchable_path = searchable_path + "/"
+            exclude_match = exclude.search(searchable_path)
+            if exclude_match and len(exclude_match.group()) > 0:
                 continue
 
-            yield from gen_python_files_in_dir(child)
+            yield from gen_python_files_in_dir(child, include, exclude)
 
-        elif child.is_file() and child.suffix in PYTHON_EXTENSIONS:
+        else:
-            yield child
+            include_match = include.search(searchable_path)
+            exclude_match = exclude.search(searchable_path)
+            if (
+                child.is_file()
+                and include_match
+                and len(include_match.group()) > 0
+                and (not exclude_match or len(exclude_match.group()) == 0)
+            ):
+                yield child
 
 
 @dataclass

tests/test_black.py

@@ -11,6 +11,7 @@
 from typing import Any, List, Tuple, Iterator
 import unittest
 from unittest.mock import patch
+import re
 
 from click import unstyle
 from click.testing import CliRunner
@@ -851,6 +852,53 @@ def test_pipe_force_py36(self) -> None:
         actual = result.output
         self.assertFormatEqual(actual, expected)
 
+    def test_include_exclude(self) -> None:
+        path = THIS_DIR / "include_exclude_tests"
+        include = re.compile(r"\.pyi?$")
+        exclude = re.compile(r"/exclude/|/\.definitely_exclude/")
+        sources: List[Path] = []
+        expected = [
+            Path(THIS_DIR / "include_exclude_tests/b/dont_exclude/a.py"),
+            Path(THIS_DIR / "include_exclude_tests/b/dont_exclude/a.pyi"),
+        ]
+        sources.extend(black.gen_python_files_in_dir(path, include, exclude))
+        self.assertEqual(sorted(expected), sorted(sources))
+
+    def test_empty_include(self) -> None:
+        path = THIS_DIR / "include_exclude_tests"
+        empty = re.compile(r"")
+        sources: List[Path] = []
+        sources.extend(
+            black.gen_python_files_in_dir(
+                path, empty, re.compile(black.DEFAULT_EXCLUDES)
+            )
+        )
+        self.assertEqual([], (sources))
+
+    def test_empty_exclude(self) -> None:
+        path = THIS_DIR / "include_exclude_tests"
+        empty = re.compile(r"")
+        sources: List[Path] = []
+        expected = [
+            Path(THIS_DIR / "include_exclude_tests/b/dont_exclude/a.py"),
+            Path(THIS_DIR / "include_exclude_tests/b/dont_exclude/a.pyi"),
+            Path(THIS_DIR / "include_exclude_tests/b/exclude/a.py"),
+            Path(THIS_DIR / "include_exclude_tests/b/exclude/a.pyi"),
+            Path(THIS_DIR / "include_exclude_tests/b/.definitely_exclude/a.py"),
+            Path(THIS_DIR / "include_exclude_tests/b/.definitely_exclude/a.pyi"),
+        ]
+        sources.extend(
+            black.gen_python_files_in_dir(
+                path, re.compile(black.DEFAULT_INCLUDES), empty
+            )
+        )
+        self.assertEqual(sorted(expected), sorted(sources))
+
+    def test_invalid_include_exclude(self) -> None:
+        for option in ["--include", "--exclude"]:
+            result = CliRunner().invoke(black.main, ["-", option, "**()(!!*)"])
+            self.assertEqual(result.exit_code, 2)
+
 
 if __name__ == "__main__":
     unittest.main()