Implement Black Version Gallery (#1294)

Closes #1290.

gallery/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.8.2-slim
+
+RUN apt-get update
+RUN apt-get upgrade -y
+RUN apt-get install git apt-utils -y
+
+RUN git config --global user.email "black@psf.github.com"
+RUN git config --global user.name "Gallery/Black"
+
+COPY gallery.py /
+ENTRYPOINT ["python", "/gallery.py"]

gallery/README.md

@@ -0,0 +1,38 @@
+# Gallery
+
+Gallery is a script that automates process of applying different black versions to a
+selected PyPI package and seeing the results between black versions.
+
+## Build
+
+```console
+$ [sudo] docker build -t black_gallery .
+```
+
+## Run
+
+```console
+$ sudo docker run -it -v /host/output:/output -v /host/input:/input black_gallery:latest [args]
+```
+
+```
+usage: gallery.py [-h] -p PYPI_PACKAGE -b BLACK_REPO [-v VERSION] [-i INPUT] [-o OUTPUT] [versions ...]
+
+Black Gallery is a script that automates process of applying different black versions to a selected PyPI package and seeing the results between versions.
+
+positional arguments:
+  versions
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -p PYPI_PACKAGE, --pypi-package PYPI_PACKAGE
+                        PyPI package to download.
+  -b BLACK_REPO, --black-repo BLACK_REPO
+                        Black's git repository.
+  -v VERSION, --version VERSION
+                        Version for PyPI given pypi package.
+  -i INPUT, --input INPUT
+                        Input directory to read configurations.
+  -o OUTPUT, --output OUTPUT
+                        Output directory to download and put result artifacts.
+```

gallery/gallery.py

@@ -0,0 +1,305 @@
+import atexit
+import json
+import subprocess
+import tarfile
+import tempfile
+import traceback
+import venv
+import zipfile
+from argparse import ArgumentParser, Namespace
+from concurrent.futures import ThreadPoolExecutor
+from functools import lru_cache, partial
+from pathlib import Path
+from typing import (  # type: ignore # typing can't see Literal
+    Generator,
+    List,
+    Literal,
+    NamedTuple,
+    Optional,
+    Tuple,
+    Union,
+    cast,
+)
+from urllib.request import urlopen, urlretrieve
+
+PYPI_INSTANCE = "https://pypi.org/pypi"
+PYPI_TOP_PACKAGES = (
+    "https://hugovk.github.io/top-pypi-packages/top-pypi-packages-{days}-days.json"
+)
+INTERNAL_BLACK_REPO = f"{tempfile.gettempdir()}/__black"
+
+ArchiveKind = Union[tarfile.TarFile, zipfile.ZipFile]
+Days = Union[Literal[30], Literal[365]]
+
+subprocess.run = partial(subprocess.run, check=True)  # type: ignore
+# https://github.com/python/mypy/issues/1484
+
+
+class BlackVersion(NamedTuple):
+    version: str
+    config: Optional[str] = None
+
+
+def get_pypi_download_url(package: str, version: Optional[str]) -> str:
+    with urlopen(PYPI_INSTANCE + f"/{package}/json") as page:
+        metadata = json.load(page)
+
+    if version is None:
+        sources = metadata["urls"]
+    else:
+        if version in metadata["releases"]:
+            sources = metadata["releases"][version]
+        else:
+            raise ValueError(
+                f"No releases found with given version ('{version}') tag. "
+                f"Found releases: {metadata['releases'].keys()}"
+            )
+
+    for source in sources:
+        if source["python_version"] == "source":
+            break
+    else:
+        raise ValueError(f"Couldn't find any sources for {package}")
+
+    return cast(str, source["url"])
+
+
+def get_top_packages(days: Days) -> List[str]:
+    with urlopen(PYPI_TOP_PACKAGES.format(days=days)) as page:
+        result = json.load(page)
+
+    return [package["project"] for package in result["rows"]]
+
+
+def get_package_source(package: str, version: Optional[str]) -> str:
+    if package == "cpython":
+        if version is None:
+            version = "master"
+        return f"https://github.com/python/cpython/archive/{version}.zip"
+    elif package == "pypy":
+        if version is None:
+            version = "branch/default"
+        return (
+            f"https://foss.heptapod.net/pypy/pypy/repository/{version}/archive.tar.bz2"
+        )
+    else:
+        return get_pypi_download_url(package, version)
+
+
+def get_archive_manager(local_file: str) -> ArchiveKind:
+    if tarfile.is_tarfile(local_file):
+        return tarfile.open(local_file)
+    elif zipfile.is_zipfile(local_file):
+        return zipfile.ZipFile(local_file)
+    else:
+        raise ValueError("Unknown archive kind.")
+
+
+def get_first_archive_member(archive: ArchiveKind) -> str:
+    if isinstance(archive, tarfile.TarFile):
+        return archive.getnames()[0]
+    elif isinstance(archive, zipfile.ZipFile):
+        return archive.namelist()[0]
+
+
+def download_and_extract(package: str, version: Optional[str], directory: Path) -> Path:
+    source = get_package_source(package, version)
+
+    local_file, _ = urlretrieve(source, directory / f"{package}-src")
+    with get_archive_manager(local_file) as archive:
+        archive.extractall(path=directory)
+        result_dir = get_first_archive_member(archive)
+    return directory / result_dir
+
+
+def get_package(
+    package: str, version: Optional[str], directory: Path
+) -> Optional[Path]:
+    try:
+        return download_and_extract(package, version, directory)
+    except Exception:
+        print(f"Caught an exception while downloading {package}.")
+        traceback.print_exc()
+        return None
+
+
+DEFAULT_SLICE = slice(None)  # for flake8
+
+
+def download_and_extract_top_packages(
+    directory: Path, days: Days = 365, workers: int = 8, limit: slice = DEFAULT_SLICE,
+) -> Generator[Path, None, None]:
+    with ThreadPoolExecutor(max_workers=workers) as executor:
+        bound_downloader = partial(get_package, version=None, directory=directory)
+        for package in executor.map(bound_downloader, get_top_packages(days)[limit]):
+            if package is not None:
+                yield package
+
+
+def git_create_repository(repo: Path) -> None:
+    subprocess.run(["git", "init"], cwd=repo)
+    git_add_and_commit(msg="Inital commit", repo=repo)
+
+
+def git_add_and_commit(msg: str, repo: Path) -> None:
+    subprocess.run(["git", "add", "."], cwd=repo)
+    subprocess.run(["git", "commit", "-m", msg, "--allow-empty"], cwd=repo)
+
+
+def git_switch_branch(
+    branch: str, repo: Path, new: bool = False, from_branch: Optional[str] = None
+) -> None:
+    args = ["git", "checkout"]
+    if new:
+        args.append("-b")
+    args.append(branch)
+    if from_branch:
+        args.append(from_branch)
+    subprocess.run(args, cwd=repo)
+
+
+def init_repos(options: Namespace) -> Tuple[Path, ...]:
+    options.output.mkdir(exist_ok=True)
+
+    if options.top_packages:
+        source_directories = tuple(
+            download_and_extract_top_packages(
+                directory=options.output,
+                workers=options.workers,
+                limit=slice(None, options.top_packages),
+            )
+        )
+    else:
+        source_directories = (
+            download_and_extract(
+                package=options.pypi_package,
+                version=options.version,
+                directory=options.output,
+            ),
+        )
+
+    for source_directory in source_directories:
+        git_create_repository(source_directory)
+
+    if options.black_repo is None:
+        subprocess.run(
+            ["git", "clone", "https://github.com/psf/black.git", INTERNAL_BLACK_REPO],
+            cwd=options.output,
+        )
+        options.black_repo = options.output / INTERNAL_BLACK_REPO
+
+    return source_directories
+
+
+@lru_cache(8)
+def black_runner(version: str, black_repo: Path) -> Path:
+    directory = tempfile.TemporaryDirectory()
+    venv.create(directory.name, with_pip=True)
+
+    python = Path(directory.name) / "bin" / "python"
+    subprocess.run([python, "-m", "pip", "install", "-e", black_repo])
+
+    atexit.register(directory.cleanup)
+    return python
+
+
+def format_repo_with_version(
+    repo: Path,
+    from_branch: Optional[str],
+    black_repo: Path,
+    black_version: BlackVersion,
+    input_directory: Path,
+) -> str:
+    current_branch = f"black-{black_version.version}"
+    git_switch_branch(black_version.version, repo=black_repo)
+    git_switch_branch(current_branch, repo=repo, new=True, from_branch=from_branch)
+
+    format_cmd: List[Union[Path, str]] = [
+        black_runner(black_version.version, black_repo),
+        (black_repo / "black.py").resolve(),
+        ".",
+    ]
+    if black_version.config:
+        format_cmd.extend(["--config", input_directory / black_version.config])
+
+    subprocess.run(format_cmd, cwd=repo, check=False)  # ensure the process
+    # continuess to run even it can't format some files. Reporting those
+    # should be enough
+    git_add_and_commit(f"Format with black:{black_version.version}", repo=repo)
+
+    return current_branch
+
+
+def format_repos(repos: Tuple[Path, ...], options: Namespace) -> None:
+    black_versions = tuple(
+        BlackVersion(*version.split(":")) for version in options.versions
+    )
+
+    for repo in repos:
+        from_branch = None
+        for black_version in black_versions:
+            from_branch = format_repo_with_version(
+                repo=repo,
+                from_branch=from_branch,
+                black_repo=options.black_repo,
+                black_version=black_version,
+                input_directory=options.input,
+            )
+        git_switch_branch("master", repo=repo)
+
+    git_switch_branch("master", repo=options.black_repo)
+
+
+def main() -> None:
+    parser = ArgumentParser(
+        description="""Black Gallery is a script that
+    automates process of applying different black versions to a selected
+    PyPI package and seeing the results between versions."""
+    )
+
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("-p", "--pypi-package", help="PyPI package to download.")
+    group.add_argument(
+        "-t", "--top-packages", help="Top n PyPI package to download.", type=int
+    )
+
+    parser.add_argument("-b", "--black-repo", help="Black's git repository.", type=Path)
+    parser.add_argument(
+        "-v",
+        "--version",
+        help=(
+            "Version for PyPI given pypi package. "
+            "Will be discarded if used with -t option."
+        ),
+    )
+    parser.add_argument(
+        "-w",
+        "--workers",
+        help=(
+            "Maximum amount of threads to download at the sametime. "
+            "Will be discard if used with -p option."
+        ),
+    )
+    parser.add_argument(
+        "-i",
+        "--input",
+        default=Path("/input"),
+        type=Path,
+        help="Input directory to read configurations.",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        default=Path("/output"),
+        type=Path,
+        help="Output directory to download and put result artifacts.",
+    )
+    parser.add_argument("versions", nargs="*", default=("master",), help="")
+
+    options = parser.parse_args()
+    repos = init_repos(options)
+    format_repos(repos, options)
+
+
+if __name__ == "__main__":
+    main()