Delay worker count determination
os.cpu_count() can return None (sounds like a super arcane edge case
though) so the type annotation for the `workers` parameter of
`black.main` is wrong. This *could* technically cause a runtime
TypeError since it'd trip one of mypyc's runtime type checks so we
might as well fix it.
Reading the documentation (and cross-checking with the source code),
you are actually allowed to pass None as `max_workers` to
`concurrent.futures.ProcessPoolExecutor`. If it is None, the pool
initializer will simply call os.cpu_count() [^1] (defaulting to 1 if it
returns None [^2]). It'll even round down the worker count to a level
that's safe for Windows.
... so theoretically we don't even need to call os.cpu_count()
ourselves, but our Windows limit is 60 (unlike the stdlib's 61) and I'd
prefer not accidentally reintroducing a crash on machines with many,
many CPU cores.
[^1]: https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ProcessPoolExecutor
[^2]: a372a7d653/Lib/concurrent/futures/process.py (L600)
This commit is contained in:
parent
afed2c0190
commit
c0cc19b5b3
@ -1,6 +1,5 @@
|
|||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
import os
|
|
||||||
import platform
|
import platform
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
@ -28,11 +27,6 @@
|
|||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
|
|
||||||
if sys.version_info >= (3, 8):
|
|
||||||
from typing import Final
|
|
||||||
else:
|
|
||||||
from typing_extensions import Final
|
|
||||||
|
|
||||||
import click
|
import click
|
||||||
from click.core import ParameterSource
|
from click.core import ParameterSource
|
||||||
from mypy_extensions import mypyc_attr
|
from mypy_extensions import mypyc_attr
|
||||||
@ -92,7 +86,6 @@
|
|||||||
from blib2to3.pytree import Leaf, Node
|
from blib2to3.pytree import Leaf, Node
|
||||||
|
|
||||||
COMPILED = Path(__file__).suffix in (".pyd", ".so")
|
COMPILED = Path(__file__).suffix in (".pyd", ".so")
|
||||||
DEFAULT_WORKERS: Final = os.cpu_count()
|
|
||||||
|
|
||||||
# types
|
# types
|
||||||
FileContent = str
|
FileContent = str
|
||||||
@ -371,9 +364,8 @@ def validate_regex(
|
|||||||
"-W",
|
"-W",
|
||||||
"--workers",
|
"--workers",
|
||||||
type=click.IntRange(min=1),
|
type=click.IntRange(min=1),
|
||||||
default=DEFAULT_WORKERS,
|
default=None,
|
||||||
show_default=True,
|
help="Number of parallel workers [default: number of CPUs in the system]",
|
||||||
help="Number of parallel workers",
|
|
||||||
)
|
)
|
||||||
@click.option(
|
@click.option(
|
||||||
"-q",
|
"-q",
|
||||||
@ -448,7 +440,7 @@ def main( # noqa: C901
|
|||||||
extend_exclude: Optional[Pattern[str]],
|
extend_exclude: Optional[Pattern[str]],
|
||||||
force_exclude: Optional[Pattern[str]],
|
force_exclude: Optional[Pattern[str]],
|
||||||
stdin_filename: Optional[str],
|
stdin_filename: Optional[str],
|
||||||
workers: int,
|
workers: Optional[int],
|
||||||
src: Tuple[str, ...],
|
src: Tuple[str, ...],
|
||||||
config: Optional[str],
|
config: Optional[str],
|
||||||
) -> None:
|
) -> None:
|
||||||
|
@ -6,6 +6,7 @@
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import signal
|
import signal
|
||||||
import sys
|
import sys
|
||||||
from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor
|
from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor
|
||||||
@ -15,7 +16,7 @@
|
|||||||
|
|
||||||
from mypy_extensions import mypyc_attr
|
from mypy_extensions import mypyc_attr
|
||||||
|
|
||||||
from black import DEFAULT_WORKERS, WriteBack, format_file_in_place
|
from black import WriteBack, format_file_in_place
|
||||||
from black.cache import Cache, filter_cached, read_cache, write_cache
|
from black.cache import Cache, filter_cached, read_cache, write_cache
|
||||||
from black.mode import Mode
|
from black.mode import Mode
|
||||||
from black.output import err
|
from black.output import err
|
||||||
@ -87,13 +88,13 @@ def reformat_many(
|
|||||||
maybe_install_uvloop()
|
maybe_install_uvloop()
|
||||||
|
|
||||||
executor: Executor
|
executor: Executor
|
||||||
worker_count = workers if workers is not None else DEFAULT_WORKERS
|
if workers is None:
|
||||||
|
workers = os.cpu_count() or 1
|
||||||
if sys.platform == "win32":
|
if sys.platform == "win32":
|
||||||
# Work around https://bugs.python.org/issue26903
|
# Work around https://bugs.python.org/issue26903
|
||||||
assert worker_count is not None
|
workers = min(workers, 60)
|
||||||
worker_count = min(worker_count, 60)
|
|
||||||
try:
|
try:
|
||||||
executor = ProcessPoolExecutor(max_workers=worker_count)
|
executor = ProcessPoolExecutor(max_workers=workers)
|
||||||
except (ImportError, NotImplementedError, OSError):
|
except (ImportError, NotImplementedError, OSError):
|
||||||
# we arrive here if the underlying system does not support multi-processing
|
# we arrive here if the underlying system does not support multi-processing
|
||||||
# like in AWS Lambda or Termux, in which case we gracefully fallback to
|
# like in AWS Lambda or Termux, in which case we gracefully fallback to
|
||||||
|
Loading…
Reference in New Issue
Block a user