Make the use of a ThreadPoolExecutor explicit

This commit is contained in:
Łukasz Langa 2020-05-08 16:06:05 +02:00
parent 9104ebe5ae
commit c5f87fb4ef
No known key found for this signature in database
GPG Key ID: B26995E310250568

View File

@ -2,7 +2,7 @@
import asyncio import asyncio
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections import defaultdict from collections import defaultdict
from concurrent.futures import Executor, ProcessPoolExecutor from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor
from contextlib import contextmanager from contextlib import contextmanager
from datetime import datetime from datetime import datetime
from enum import Enum from enum import Enum
@ -613,6 +613,7 @@ def reformat_many(
sources: Set[Path], fast: bool, write_back: WriteBack, mode: Mode, report: "Report" sources: Set[Path], fast: bool, write_back: WriteBack, mode: Mode, report: "Report"
) -> None: ) -> None:
"""Reformat multiple files using a ProcessPoolExecutor.""" """Reformat multiple files using a ProcessPoolExecutor."""
executor: Executor
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
worker_count = os.cpu_count() worker_count = os.cpu_count()
if sys.platform == "win32": if sys.platform == "win32":
@ -622,9 +623,10 @@ def reformat_many(
executor = ProcessPoolExecutor(max_workers=worker_count) executor = ProcessPoolExecutor(max_workers=worker_count)
except OSError: except OSError:
# we arrive here if the underlying system does not support multi-processing # we arrive here if the underlying system does not support multi-processing
# like in AWS Lambda, in which case we gracefully fallback to the default # like in AWS Lambda, in which case we gracefully fallback to
# mono-process Executor by using None # a ThreadPollExecutor with just a single worker (more workers would not do us
executor = None # any good due to the Global Interpreter Lock)
executor = ThreadPoolExecutor(max_workers=1)
try: try:
loop.run_until_complete( loop.run_until_complete(
@ -651,7 +653,7 @@ async def schedule_formatting(
mode: Mode, mode: Mode,
report: "Report", report: "Report",
loop: asyncio.AbstractEventLoop, loop: asyncio.AbstractEventLoop,
executor: Optional[Executor], executor: Executor,
) -> None: ) -> None:
"""Run formatting of `sources` in parallel using the provided `executor`. """Run formatting of `sources` in parallel using the provided `executor`.