don't run more than 61 workers on Windows (#838)

This commit is contained in:
Jelle Zijlstra 2019-05-07 13:11:20 -04:00 committed by GitHub
parent 3b2297f6fd
commit 14cbf737df
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 36 additions and 16 deletions

View File

@ -986,6 +986,8 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
### 19.5b0
* don't crash when run on a Windows machine with more than 61 cores (#838)
* remove unnecessary parentheses around `yield` expressions (#834)
* add parentheses around long tuples in unpacking assignments (#832)

View File

@ -440,22 +440,10 @@ def main(
report=report,
)
else:
loop = asyncio.get_event_loop()
executor = ProcessPoolExecutor(max_workers=os.cpu_count())
try:
loop.run_until_complete(
schedule_formatting(
sources=sources,
fast=fast,
write_back=write_back,
mode=mode,
report=report,
loop=loop,
executor=executor,
)
)
finally:
shutdown(loop)
reformat_many(
sources=sources, fast=fast, write_back=write_back, mode=mode, report=report
)
if verbose or not quiet:
bang = "💥 💔 💥" if report.return_code else "✨ 🍰 ✨"
out(f"All done! {bang}")
@ -497,6 +485,36 @@ def reformat_one(
report.failed(src, str(exc))
def reformat_many(
sources: Set[Path],
fast: bool,
write_back: WriteBack,
mode: FileMode,
report: "Report",
) -> None:
"""Reformat multiple files using a ProcessPoolExecutor."""
loop = asyncio.get_event_loop()
worker_count = os.cpu_count()
if sys.platform == "win32":
# Work around https://bugs.python.org/issue26903
worker_count = min(worker_count, 61)
executor = ProcessPoolExecutor(max_workers=worker_count)
try:
loop.run_until_complete(
schedule_formatting(
sources=sources,
fast=fast,
write_back=write_back,
mode=mode,
report=report,
loop=loop,
executor=executor,
)
)
finally:
shutdown(loop)
async def schedule_formatting(
sources: Set[Path],
fast: bool,