Convert legacy string formatting to f-strings (#4685)

* the changes

* Update driver.py

.git_archival.txt

@@ -1,4 +1,3 @@
 node: $Format:%H$
 node-date: $Format:%cI$
-describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$
-ref-names: $Format:%D$
+describe-name: $Format:%(describe:tags=true,match=[0-9]*)$

.github/workflows/diff_shades.yml

@@ -34,7 +34,8 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ github.token }}
         run: >
-          python scripts/diff_shades_gha_helper.py config ${{ github.event_name }} ${{ matrix.mode }}
+          python scripts/diff_shades_gha_helper.py config ${{ github.event_name }}
+          ${{ matrix.mode }}
 
   analysis:
     name: analysis / ${{ matrix.mode }}
@@ -48,7 +49,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJson(needs.configure.outputs.matrix )}}
+        include: ${{ fromJson(needs.configure.outputs.matrix) }}
 
     steps:
       - name: Checkout this repository (full clone)
@@ -110,19 +111,19 @@ jobs:
           ${{ matrix.baseline-analysis }} ${{ matrix.target-analysis }}
 
       - name: Upload diff report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.mode }}-diff.html
           path: diff.html
 
       - name: Upload baseline analysis
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.baseline-analysis }}
           path: ${{ matrix.baseline-analysis }}
 
       - name: Upload target analysis
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.target-analysis }}
           path: ${{ matrix.target-analysis }}
@@ -130,14 +131,13 @@ jobs:
       - name: Generate summary file (PR only)
         if: github.event_name == 'pull_request' && matrix.mode == 'preview-changes'
         run: >
-          python helper.py comment-body
-          ${{ matrix.baseline-analysis }} ${{ matrix.target-analysis }}
-          ${{ matrix.baseline-sha }} ${{ matrix.target-sha }}
-          ${{ github.event.pull_request.number }}
+          python helper.py comment-body ${{ matrix.baseline-analysis }}
+          ${{ matrix.target-analysis }} ${{ matrix.baseline-sha }}
+          ${{ matrix.target-sha }} ${{ github.event.pull_request.number }}
 
       - name: Upload summary file (PR only)
         if: github.event_name == 'pull_request' && matrix.mode == 'preview-changes'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: .pr-comment.json
           path: .pr-comment.json

.github/workflows/pypi_upload.yml

@@ -92,7 +92,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       # Keep cibuildwheel version in sync with above
-      - uses: pypa/cibuildwheel@v2.22.0
+      - uses: pypa/cibuildwheel@v2.23.3
         with:
           only: ${{ matrix.only }}
 

.github/workflows/upload_binary.yml

@@ -13,13 +13,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-2019, ubuntu-20.04, macos-latest]
+        os: [windows-2019, ubuntu-22.04, macos-latest]
         include:
           - os: windows-2019
             pathsep: ";"
             asset_name: black_windows.exe
             executable_mime: "application/vnd.microsoft.portable-executable"
-          - os: ubuntu-20.04
+          - os: ubuntu-22.04
             pathsep: ":"
             asset_name: black_linux
             executable_mime: "application/x-executable"

.pre-commit-config.yaml

@@ -24,12 +24,12 @@ repos:
         additional_dependencies: *version_check_dependencies
 
   - repo: https://github.com/pycqa/isort
-    rev: 5.13.2
+    rev: 6.0.1
     hooks:
       - id: isort
 
   - repo: https://github.com/pycqa/flake8
-    rev: 7.1.1
+    rev: 7.2.0
     hooks:
       - id: flake8
         additional_dependencies:
@@ -39,17 +39,21 @@ repos:
         exclude: ^src/blib2to3/
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.14.1
+    rev: v1.15.0
     hooks:
       - id: mypy
         exclude: ^(docs/conf.py|scripts/generate_schema.py)$
         args: []
         additional_dependencies: &mypy_deps
           - types-PyYAML
+          - types-atheris
           - tomli >= 0.2.6, < 2.0.0
-          - click >= 8.1.0, != 8.1.4, != 8.1.5
+          - click >= 8.2.0
+          # Click is intentionally out-of-sync with pyproject.toml
+          # v8.2 has breaking changes. We work around them at runtime, but we need the newer stubs.
           - packaging >= 22.0
           - platformdirs >= 2.1.0
+          - pytokens >= 0.1.10
           - pytest
           - hypothesis
           - aiohttp >= 3.7.4
@@ -62,11 +66,11 @@ repos:
         args: ["--python-version=3.10"]
         additional_dependencies: *mypy_deps
 
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v4.0.0-alpha.8
+  - repo: https://github.com/rbubley/mirrors-prettier
+    rev: v3.5.3
     hooks:
       - id: prettier
-        types_or: [css, javascript, html, json, yaml]
+        types_or: [markdown, yaml, json]
         exclude: \.github/workflows/diff_shades\.yml
 
   - repo: https://github.com/pre-commit/pre-commit-hooks

CHANGES.md

@@ -1,11 +1,80 @@
 # Change Log
 
+## Unreleased
+
+### Highlights
+
+<!-- Include any especially major or disruptive changes here -->
+
+### Stable style
+
+<!-- Changes that affect Black's stable style -->
+
+- Fix crash while formatting a long `del` statement containing tuples (#4628)
+- Fix crash while formatting expressions using the walrus operator in complex `with`
+  statements (#4630)
+- Handle `# fmt: skip` followed by a comment at the end of file (#4635)
+- Fix crash when a tuple appears in the `as` clause of a `with` statement (#4634)
+- Fix crash when tuple is used as a context manager inside a `with` statement (#4646)
+- Fix crash on a `\\r\n` (#4673)
+- Fix crash on `await ...` (where `...` is a literal `Ellipsis`) (#4676)
+- Remove support for pre-python 3.7 `await/async` as soft keywords/variable names
+  (#4676)
+
+### Preview style
+
+<!-- Changes that affect Black's preview style -->
+
+- Fix a bug where one-liner functions/conditionals marked with `# fmt: skip` would still
+  be formatted (#4552)
+
+### Configuration
+
+<!-- Changes to how Black can be configured -->
+
+### Packaging
+
+<!-- Changes to how Black is packaged, such as dependency requirements -->
+
+### Parser
+
+<!-- Changes to the parser or to version autodetection -->
+
+- Rewrite tokenizer to improve performance and compliance (#4536)
+- Fix bug where certain unusual expressions (e.g., lambdas) were not accepted in type
+  parameter bounds and defaults. (#4602)
+
+### Performance
+
+<!-- Changes that improve Black's performance. -->
+
+### Output
+
+<!-- Changes to Black's terminal output and error messages -->
+
+### _Blackd_
+
+<!-- Changes to blackd -->
+
+### Integrations
+
+<!-- For example, Docker, GitHub Actions, pre-commit, editors -->
+
+- Fix the version check in the vim file to reject Python 3.8 (#4567)
+- Enhance GitHub Action `psf/black` to read Black version from an additional section in
+  pyproject.toml: `[project.dependency-groups]` (#4606)
+
+### Documentation
+
+<!-- Major changes to documentation and policies. Small docs changes
+     don't need a changelog entry. -->
+
 ## 25.1.0
 
 ### Highlights
 
-This release introduces the new 2025 stable style (#4558), stabilizing
-the following changes:
+This release introduces the new 2025 stable style (#4558), stabilizing the following
+changes:
 
 - Normalize casing of Unicode escape characters in strings to lowercase (#2916)
 - Fix inconsistencies in whether certain strings are detected as docstrings (#4095)
@@ -13,15 +82,16 @@ the following changes:
 - Remove redundant parentheses in if guards for case blocks (#4214)
 - Add parentheses to if clauses in case blocks when the line is too long (#4269)
 - Whitespace before `# fmt: skip` comments is no longer normalized (#4146)
-- Fix line length computation for certain expressions that involve the power operator (#4154)
+- Fix line length computation for certain expressions that involve the power operator
+  (#4154)
 - Check if there is a newline before the terminating quotes of a docstring (#4185)
 - Fix type annotation spacing between `*` and more complex type variable tuple (#4440)
 
 The following changes were not in any previous release:
 
 - Remove parentheses around sole list items (#4312)
-- Generic function definitions are now formatted more elegantly: parameters are
-  split over multiple lines first instead of type parameter definitions (#4553)
+- Generic function definitions are now formatted more elegantly: parameters are split
+  over multiple lines first instead of type parameter definitions (#4553)
 
 ### Stable style
 

README.md

@@ -137,8 +137,8 @@ SQLAlchemy, Poetry, PyPA applications (Warehouse, Bandersnatch, Pipenv, virtuale
 pandas, Pillow, Twisted, LocalStack, every Datadog Agent Integration, Home Assistant,
 Zulip, Kedro, OpenOA, FLORIS, ORBIT, WOMBAT, and many more.
 
-The following organizations use _Black_: Dropbox, KeepTruckin, Lyft, Mozilla,
-Quora, Duolingo, QuantumBlack, Tesla, Archer Aviation.
+The following organizations use _Black_: Dropbox, KeepTruckin, Lyft, Mozilla, Quora,
+Duolingo, QuantumBlack, Tesla, Archer Aviation.
 
 Are we missing anyone? Let us know.
 

action/main.py

@@ -71,6 +71,7 @@ def read_version_specifier_from_pyproject() -> str:
         return f"=={version}"
 
     arrays = [
+        *pyproject.get("dependency-groups", {}).values(),
         pyproject.get("project", {}).get("dependencies"),
         *pyproject.get("project", {}).get("optional-dependencies", {}).values(),
     ]

autoload/black.vim

@@ -75,7 +75,7 @@ def _initialize_black_env(upgrade=False):
       return True
 
   pyver = sys.version_info[:3]
-  if pyver < (3, 8):
+  if pyver < (3, 9):
     print("Sorry, Black requires Python 3.9+ to run.")
     return False
 

docs/contributing/release_process.md

@@ -29,8 +29,8 @@ frequently than monthly nets rapidly diminishing returns.
 **You must have `write` permissions for the _Black_ repository to cut a release.**
 
 The 10,000 foot view of the release process is that you prepare a release PR and then
-publish a [GitHub Release]. This triggers [release automation](#release-workflows) that builds
-all release artifacts and publishes them to the various platforms we publish to.
+publish a [GitHub Release]. This triggers [release automation](#release-workflows) that
+builds all release artifacts and publishes them to the various platforms we publish to.
 
 We now have a `scripts/release.py` script to help with cutting the release PRs.
 
@@ -96,8 +96,9 @@ In the end, use your best judgement and ask other maintainers for their thoughts
 
 ## Release workflows
 
-All of _Black_'s release automation uses [GitHub Actions]. All workflows are therefore configured
-using YAML files in the `.github/workflows` directory of the _Black_ repository.
+All of _Black_'s release automation uses [GitHub Actions]. All workflows are therefore
+configured using YAML files in the `.github/workflows` directory of the _Black_
+repository.
 
 They are triggered by the publication of a [GitHub Release].
 

docs/faq.md

@@ -93,6 +93,8 @@ Support for formatting Python 2 code was removed in version 22.0. While we've ma
 plans to stop supporting older Python 3 minor versions immediately, their support might
 also be removed some time in the future without a deprecation period.
 
+`await`/`async` as soft keywords/indentifiers are no longer supported as of 25.2.0.
+
 Runtime support for 3.6 was removed in version 22.10.0, for 3.7 in version 23.7.0, and
 for 3.8 in version 24.10.0.
 

docs/integrations/github_actions.md

@@ -37,10 +37,10 @@ the `pyproject.toml` file. `version` can be any
 [valid version specifier](https://packaging.python.org/en/latest/glossary/#term-Version-Specifier)
 or just the version number if you want an exact version. To read the version from the
 `pyproject.toml` file instead, set `use_pyproject` to `true`. This will first look into
-the `tool.black.required-version` field, then the `project.dependencies` array and
-finally the `project.optional-dependencies` table. The action defaults to the latest
-release available on PyPI. Only versions available from PyPI are supported, so no commit
-SHAs or branch names.
+the `tool.black.required-version` field, then the `dependency-groups` table, then the
+`project.dependencies` array and finally the `project.optional-dependencies` table. The
+action defaults to the latest release available on PyPI. Only versions available from
+PyPI are supported, so no commit SHAs or branch names.
 
 If you want to include Jupyter Notebooks, _Black_ must be installed with the `jupyter`
 extra. Installing the extra and including Jupyter Notebook files can be configured via

docs/requirements.txt

@@ -1,7 +1,7 @@
 # Used by ReadTheDocs; pinned requirements for stability.
 
-myst-parser==4.0.0
-Sphinx==8.1.3
+myst-parser==4.0.1
+Sphinx==8.2.3
 # Older versions break Sphinx even though they're declared to be supported.
 docutils==0.21.2
 sphinxcontrib-programoutput==0.18

docs/the_black_code_style/future_style.md

@@ -26,6 +26,9 @@ Currently, the following features are included in the preview style:
   statements, except when the line after the import is a comment or an import statement
 - `wrap_long_dict_values_in_parens`: Add parentheses around long values in dictionaries
   ([see below](labels/wrap-long-dict-values))
+- `fix_fmt_skip_in_one_liners`: Fix `# fmt: skip` behaviour on one-liner declarations,
+  such as `def foo(): return "mock"  # fmt: skip`, where previously the declaration
+  would have been incorrectly collapsed.
 
 (labels/unstable-features)=
 

gallery/gallery.py

@@ -16,7 +16,7 @@
 
 PYPI_INSTANCE = "https://pypi.org/pypi"
 PYPI_TOP_PACKAGES = (
-    "https://hugovk.github.io/top-pypi-packages/top-pypi-packages-30-days.min.json"
+    "https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json"
 )
 INTERNAL_BLACK_REPO = f"{tempfile.gettempdir()}/__black"
 

pyproject.toml

@@ -69,6 +69,7 @@ dependencies = [
   "packaging>=22.0",
   "pathspec>=0.9.0",
   "platformdirs>=2",
+  "pytokens>=0.1.10",
   "tomli>=1.1.0; python_version < '3.11'",
   "typing_extensions>=4.0.1; python_version < '3.11'",
 ]
@@ -186,16 +187,6 @@ MYPYC_DEBUG_LEVEL = "0"
 # Black needs Clang to compile successfully on Linux.
 CC = "clang"
 
-[tool.cibuildwheel.macos]
-build-frontend = { name = "build", args = ["--no-isolation"] }
-# Unfortunately, hatch doesn't respect MACOSX_DEPLOYMENT_TARGET
-# Note we don't have a good test for this sed horror, so if you futz with it
-# make sure to test manually
-before-build = [
-    "python -m pip install 'hatchling==1.20.0' hatch-vcs hatch-fancy-pypi-readme 'hatch-mypyc>=0.16.0' 'mypy>=1.12' 'click>=8.1.7'",
-    """sed -i '' -e "600,700s/'10_16'/os.environ['MACOSX_DEPLOYMENT_TARGET'].replace('.', '_')/" $(python -c 'import hatchling.builders.wheel as h; print(h.__file__)') """,
-]
-
 [tool.isort]
 atomic = true
 profile = "black"
@@ -234,6 +225,8 @@ branch = true
 python_version = "3.9"
 mypy_path = "src"
 strict = true
+strict_bytes = true
+local_partial_types = true
 # Unreachable blocks have been an issue when compiling mypyc, let's try to avoid 'em in the first place.
 warn_unreachable = true
 implicit_reexport = true

scripts/fuzz.py

@@ -5,14 +5,11 @@
 a coverage-guided fuzzer I'm working on.
 """
 
-import re
-
 import hypothesmith
 from hypothesis import HealthCheck, given, settings
 from hypothesis import strategies as st
 
 import black
-from blib2to3.pgen2.tokenize import TokenError
 
 
 # This test uses the Hypothesis and Hypothesmith libraries to generate random
@@ -45,23 +42,7 @@ def test_idempotent_any_syntatically_valid_python(
     compile(src_contents, "<string>", "exec")  # else the bug is in hypothesmith
 
     # Then format the code...
-    try:
-        dst_contents = black.format_str(src_contents, mode=mode)
-    except black.InvalidInput:
-        # This is a bug - if it's valid Python code, as above, Black should be
-        # able to cope with it.  See issues #970, #1012
-        # TODO: remove this try-except block when issues are resolved.
-        return
-    except TokenError as e:
-        if (  # Special-case logic for backslashes followed by newlines or end-of-input
-            e.args[0] == "EOF in multi-line statement"
-            and re.search(r"\\($|\r?\n)", src_contents) is not None
-        ):
-            # This is a bug - if it's valid Python code, as above, Black should be
-            # able to cope with it.  See issue #1012.
-            # TODO: remove this block when the issue is resolved.
-            return
-        raise
+    dst_contents = black.format_str(src_contents, mode=mode)
 
     # And check that we got equivalent and stable output.
     black.assert_equivalent(src_contents, dst_contents)
@@ -80,7 +61,7 @@ def test_idempotent_any_syntatically_valid_python(
     try:
         import sys
 
-        import atheris  # type: ignore[import-not-found]
+        import atheris
     except ImportError:
         pass
     else:

scripts/migrate-black.py

@@ -77,7 +77,7 @@ def blackify(base_branch: str, black_command: str, logger: logging.Logger) -> in
         git("commit", "--allow-empty", "-aqC", commit)
 
     for commit in commits:
-        git("branch", "-qD", "%s-black" % commit)
+        git("branch", "-qD", f"{commit}-black")
 
     return 0
 

src/black/comments.py

@@ -4,7 +4,7 @@
 from functools import lru_cache
 from typing import Final, Optional, Union
 
-from black.mode import Mode
+from black.mode import Mode, Preview
 from black.nodes import (
     CLOSING_BRACKETS,
     STANDALONE_COMMENT,
@@ -270,7 +270,7 @@ def generate_ignored_nodes(
     Stops at the end of the block.
     """
     if _contains_fmt_skip_comment(comment.value, mode):
-        yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment)
+        yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment, mode)
         return
     container: Optional[LN] = container_of(leaf)
     while container is not None and container.type != token.ENDMARKER:
@@ -309,23 +309,67 @@ def generate_ignored_nodes(
 
 
 def _generate_ignored_nodes_from_fmt_skip(
-    leaf: Leaf, comment: ProtoComment
+    leaf: Leaf, comment: ProtoComment, mode: Mode
 ) -> Iterator[LN]:
     """Generate all leaves that should be ignored by the `# fmt: skip` from `leaf`."""
     prev_sibling = leaf.prev_sibling
     parent = leaf.parent
+    ignored_nodes: list[LN] = []
     # Need to properly format the leaf prefix to compare it to comment.value,
     # which is also formatted
     comments = list_comments(leaf.prefix, is_endmarker=False)
     if not comments or comment.value != comments[0].value:
         return
     if prev_sibling is not None:
-        leaf.prefix = ""
-        siblings = [prev_sibling]
-        while "\n" not in prev_sibling.prefix and prev_sibling.prev_sibling is not None:
-            prev_sibling = prev_sibling.prev_sibling
-            siblings.insert(0, prev_sibling)
-        yield from siblings
+        leaf.prefix = leaf.prefix[comment.consumed :]
+
+        if Preview.fix_fmt_skip_in_one_liners not in mode:
+            siblings = [prev_sibling]
+            while (
+                "\n" not in prev_sibling.prefix
+                and prev_sibling.prev_sibling is not None
+            ):
+                prev_sibling = prev_sibling.prev_sibling
+                siblings.insert(0, prev_sibling)
+            yield from siblings
+            return
+
+        # Generates the nodes to be ignored by `fmt: skip`.
+
+        # Nodes to ignore are the ones on the same line as the
+        # `# fmt: skip` comment, excluding the `# fmt: skip`
+        # node itself.
+
+        # Traversal process (starting at the `# fmt: skip` node):
+        # 1. Move to the `prev_sibling` of the current node.
+        # 2. If `prev_sibling` has children, go to its rightmost leaf.
+        # 3. If there’s no `prev_sibling`, move up to the parent
+        # node and repeat.
+        # 4. Continue until:
+        #    a. You encounter an `INDENT` or `NEWLINE` node (indicates
+        #       start of the line).
+        #    b. You reach the root node.
+
+        # Include all visited LEAVES in the ignored list, except INDENT
+        # or NEWLINE leaves.
+
+        current_node = prev_sibling
+        ignored_nodes = [current_node]
+        if current_node.prev_sibling is None and current_node.parent is not None:
+            current_node = current_node.parent
+        while "\n" not in current_node.prefix and current_node.prev_sibling is not None:
+            leaf_nodes = list(current_node.prev_sibling.leaves())
+            current_node = leaf_nodes[-1] if leaf_nodes else current_node
+
+            if current_node.type in (token.NEWLINE, token.INDENT):
+                current_node.prefix = ""
+                break
+
+            ignored_nodes.insert(0, current_node)
+
+            if current_node.prev_sibling is None and current_node.parent is not None:
+                current_node = current_node.parent
+        yield from ignored_nodes
     elif (
         parent is not None and parent.type == syms.suite and leaf.type == token.NEWLINE
     ):
@@ -333,7 +377,6 @@ def _generate_ignored_nodes_from_fmt_skip(
         # statements. The ignored nodes should be previous siblings of the
         # parent suite node.
         leaf.prefix = ""
-        ignored_nodes: list[LN] = []
         parent_sibling = parent.prev_sibling
         while parent_sibling is not None and parent_sibling.type != syms.suite:
             ignored_nodes.insert(0, parent_sibling)

src/black/linegen.py

@@ -40,6 +40,7 @@
     ensure_visible,
     fstring_to_string,
     get_annotation_type,
+    has_sibling_with_type,
     is_arith_like,
     is_async_stmt_or_funcdef,
     is_atom_with_invisible_parens,
@@ -56,6 +57,7 @@
     is_rpar_token,
     is_stub_body,
     is_stub_suite,
+    is_tuple,
     is_tuple_containing_star,
     is_tuple_containing_walrus,
     is_type_ignore_comment_string,
@@ -1626,6 +1628,12 @@ def maybe_make_parens_invisible_in_atom(
         node.type not in (syms.atom, syms.expr)
         or is_empty_tuple(node)
         or is_one_tuple(node)
+        or (is_tuple(node) and parent.type == syms.asexpr_test)
+        or (
+            is_tuple(node)
+            and parent.type == syms.with_stmt
+            and has_sibling_with_type(node, token.COMMA)
+        )
         or (is_yield(node) and parent.type != syms.expr_stmt)
         or (
             # This condition tries to prevent removing non-optional brackets
@@ -1649,6 +1657,7 @@ def maybe_make_parens_invisible_in_atom(
             syms.except_clause,
             syms.funcdef,
             syms.with_stmt,
+            syms.testlist_gexp,
             syms.tname,
             # these ones aren't useful to end users, but they do please fuzzers
             syms.for_stmt,

src/black/mode.py

@@ -203,6 +203,7 @@ class Preview(Enum):
     wrap_long_dict_values_in_parens = auto()
     multiline_string_handling = auto()
     always_one_newline_after_import = auto()
+    fix_fmt_skip_in_one_liners = auto()
 
 
 UNSTABLE_FEATURES: set[Preview] = {

src/black/nodes.py

@@ -603,6 +603,17 @@ def is_one_tuple(node: LN) -> bool:
     )
 
 
+def is_tuple(node: LN) -> bool:
+    """Return True if `node` holds a tuple."""
+    if node.type != syms.atom:
+        return False
+    gexp = unwrap_singleton_parenthesis(node)
+    if gexp is None or gexp.type != syms.testlist_gexp:
+        return False
+
+    return True
+
+
 def is_tuple_containing_walrus(node: LN) -> bool:
     """Return True if `node` holds a tuple that contains a walrus operator."""
     if node.type != syms.atom:
@@ -1047,3 +1058,21 @@ def furthest_ancestor_with_last_leaf(leaf: Leaf) -> LN:
     while node.parent and node.parent.children and node is node.parent.children[-1]:
         node = node.parent
     return node
+
+
+def has_sibling_with_type(node: LN, type: int) -> bool:
+    # Check previous siblings
+    sibling = node.prev_sibling
+    while sibling is not None:
+        if sibling.type == type:
+            return True
+        sibling = sibling.prev_sibling
+
+    # Check next siblings
+    sibling = node.next_sibling
+    while sibling is not None:
+        if sibling.type == type:
+            return True
+        sibling = sibling.next_sibling
+
+    return False

src/black/parsing.py

@@ -213,7 +213,7 @@ def _stringify_ast(node: ast.AST, parent_stack: list[ast.AST]) -> Iterator[str]:
                     and isinstance(node, ast.Delete)
                     and isinstance(item, ast.Tuple)
                 ):
-                    for elt in item.elts:
+                    for elt in _unwrap_tuples(item):
                         yield from _stringify_ast_with_new_parent(
                             elt, parent_stack, node
                         )
@@ -250,3 +250,11 @@ def _stringify_ast(node: ast.AST, parent_stack: list[ast.AST]) -> Iterator[str]:
             )
 
     yield f"{'    ' * len(parent_stack)})  # /{node.__class__.__name__}"
+
+
+def _unwrap_tuples(node: ast.Tuple) -> Iterator[ast.AST]:
+    for elt in node.elts:
+        if isinstance(elt, ast.Tuple):
+            yield from _unwrap_tuples(elt)
+        else:
+            yield elt

src/black/resources/black.schema.json

@@ -83,7 +83,8 @@
           "hug_parens_with_braces_and_square_brackets",
           "wrap_long_dict_values_in_parens",
           "multiline_string_handling",
-          "always_one_newline_after_import"
+          "always_one_newline_after_import",
+          "fix_fmt_skip_in_one_liners"
         ]
       },
       "description": "Enable specific features included in the `--unstable` style. Requires `--preview`. No compatibility guarantees are provided on the behavior or existence of any unstable features."

src/blib2to3/Grammar.txt

@@ -12,9 +12,9 @@ file_input: (NEWLINE | stmt)* ENDMARKER
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
 eval_input: testlist NEWLINE* ENDMARKER
 
-typevar: NAME [':' expr] ['=' expr]
-paramspec: '**' NAME ['=' expr]
-typevartuple: '*' NAME ['=' (expr|star_expr)]
+typevar: NAME [':' test] ['=' test]
+paramspec: '**' NAME ['=' test]
+typevartuple: '*' NAME ['=' (test|star_expr)]
 typeparam: typevar | paramspec | typevartuple
 typeparams: '[' typeparam (',' typeparam)* [','] ']'
 

src/blib2to3/pgen2/driver.py

@@ -28,7 +28,7 @@
 from typing import IO, Any, Optional, Union, cast
 
 from blib2to3.pgen2.grammar import Grammar
-from blib2to3.pgen2.tokenize import GoodTokenInfo
+from blib2to3.pgen2.tokenize import TokenInfo
 from blib2to3.pytree import NL
 
 # Pgen imports
@@ -112,7 +112,7 @@ def __init__(self, grammar: Grammar, logger: Optional[Logger] = None) -> None:
             logger = logging.getLogger(__name__)
         self.logger = logger
 
-    def parse_tokens(self, tokens: Iterable[GoodTokenInfo], debug: bool = False) -> NL:
+    def parse_tokens(self, tokens: Iterable[TokenInfo], debug: bool = False) -> NL:
         """Parse a series of tokens and return the syntax tree."""
         # XXX Move the prefix computation into a wrapper around tokenize.
         proxy = TokenProxy(tokens)
@@ -180,27 +180,17 @@ def parse_tokens(self, tokens: Iterable[GoodTokenInfo], debug: bool = False) ->
         assert p.rootnode is not None
         return p.rootnode
 
-    def parse_stream_raw(self, stream: IO[str], debug: bool = False) -> NL:
-        """Parse a stream and return the syntax tree."""
-        tokens = tokenize.generate_tokens(stream.readline, grammar=self.grammar)
-        return self.parse_tokens(tokens, debug)
-
-    def parse_stream(self, stream: IO[str], debug: bool = False) -> NL:
-        """Parse a stream and return the syntax tree."""
-        return self.parse_stream_raw(stream, debug)
-
     def parse_file(
         self, filename: Path, encoding: Optional[str] = None, debug: bool = False
     ) -> NL:
         """Parse a file and return the syntax tree."""
         with open(filename, encoding=encoding) as stream:
-            return self.parse_stream(stream, debug)
+            text = stream.read()
+        return self.parse_string(text, debug)
 
     def parse_string(self, text: str, debug: bool = False) -> NL:
         """Parse a string and return the syntax tree."""
-        tokens = tokenize.generate_tokens(
-            io.StringIO(text).readline, grammar=self.grammar
-        )
+        tokens = tokenize.tokenize(text, grammar=self.grammar)
         return self.parse_tokens(tokens, debug)
 
     def _partially_consume_prefix(self, prefix: str, column: int) -> tuple[str, str]:

src/blib2to3/pgen2/literals.py

@@ -28,16 +28,16 @@ def escape(m: re.Match[str]) -> str:
     if tail.startswith("x"):
         hexes = tail[1:]
         if len(hexes) < 2:
-            raise ValueError("invalid hex string escape ('\\%s')" % tail)
+            raise ValueError(f"invalid hex string escape ('\\{tail}')")
         try:
             i = int(hexes, 16)
         except ValueError:
-            raise ValueError("invalid hex string escape ('\\%s')" % tail) from None
+            raise ValueError(f"invalid hex string escape ('\\{tail}')") from None
     else:
         try:
             i = int(tail, 8)
         except ValueError:
-            raise ValueError("invalid octal string escape ('\\%s')" % tail) from None
+            raise ValueError(f"invalid octal string escape ('\\{tail}')") from None
     return chr(i)
 
 

src/blib2to3/pgen2/parse.py

@@ -89,18 +89,12 @@ def backtrack(self) -> Iterator[None]:
             self.parser.is_backtracking = is_backtracking
 
     def add_token(self, tok_type: int, tok_val: str, raw: bool = False) -> None:
-        func: Callable[..., Any]
-        if raw:
-            func = self.parser._addtoken
-        else:
-            func = self.parser.addtoken
-
         for ilabel in self.ilabels:
             with self.switch_to(ilabel):
-                args = [tok_type, tok_val, self.context]
                 if raw:
-                    args.insert(0, ilabel)
-                func(*args)
+                    self.parser._addtoken(ilabel, tok_type, tok_val, self.context)
+                else:
+                    self.parser.addtoken(tok_type, tok_val, self.context)
 
     def determine_route(
         self, value: Optional[str] = None, force: bool = False

src/blib2to3/pgen2/pgen.py

@@ -6,7 +6,7 @@
 from typing import IO, Any, NoReturn, Optional, Union
 
 from blib2to3.pgen2 import grammar, token, tokenize
-from blib2to3.pgen2.tokenize import GoodTokenInfo
+from blib2to3.pgen2.tokenize import TokenInfo
 
 Path = Union[str, "os.PathLike[str]"]
 
@@ -18,7 +18,7 @@ class PgenGrammar(grammar.Grammar):
 class ParserGenerator:
     filename: Path
     stream: IO[str]
-    generator: Iterator[GoodTokenInfo]
+    generator: Iterator[TokenInfo]
     first: dict[str, Optional[dict[str, int]]]
 
     def __init__(self, filename: Path, stream: Optional[IO[str]] = None) -> None:
@@ -27,8 +27,7 @@ def __init__(self, filename: Path, stream: Optional[IO[str]] = None) -> None:
             stream = open(filename, encoding="utf-8")
             close_stream = stream.close
         self.filename = filename
-        self.stream = stream
-        self.generator = tokenize.generate_tokens(stream.readline)
+        self.generator = tokenize.tokenize(stream.read())
         self.gettoken()  # Initialize lookahead
         self.dfas, self.startsymbol = self.parse()
         if close_stream is not None:
@@ -141,7 +140,7 @@ def calcfirst(self, name: str) -> None:
                 if label in self.first:
                     fset = self.first[label]
                     if fset is None:
-                        raise ValueError("recursion for rule %r" % name)
+                        raise ValueError(f"recursion for rule {name!r}")
                 else:
                     self.calcfirst(label)
                     fset = self.first[label]
@@ -156,8 +155,8 @@ def calcfirst(self, name: str) -> None:
             for symbol in itsfirst:
                 if symbol in inverse:
                     raise ValueError(
-                        "rule %s is ambiguous; %s is in the first sets of %s as well"
-                        " as %s" % (name, symbol, label, inverse[symbol])
+                        f"rule {name} is ambiguous; {symbol} is in the first sets of"
+                        f" {label} as well as {inverse[symbol]}"
                     )
                 inverse[symbol] = label
         self.first[name] = totalset
@@ -238,16 +237,16 @@ def dump_nfa(self, name: str, start: "NFAState", finish: "NFAState") -> None:
                     j = len(todo)
                     todo.append(next)
                 if label is None:
-                    print("    -> %d" % j)
+                    print(f"    -> {j}")
                 else:
-                    print("    %s -> %d" % (label, j))
+                    print(f"    {label} -> {j}")
 
     def dump_dfa(self, name: str, dfa: Sequence["DFAState"]) -> None:
         print("Dump of DFA for", name)
         for i, state in enumerate(dfa):
             print("  State", i, state.isfinal and "(final)" or "")
             for label, next in sorted(state.arcs.items()):
-                print("    %s -> %d" % (label, dfa.index(next)))
+                print(f"    {label} -> {dfa.index(next)}")
 
     def simplify_dfa(self, dfa: list["DFAState"]) -> None:
         # This is not theoretically optimal, but works well enough.
@@ -331,15 +330,12 @@ def parse_atom(self) -> tuple["NFAState", "NFAState"]:
             return a, z
         else:
             self.raise_error(
-                "expected (...) or NAME or STRING, got %s/%s", self.type, self.value
+                f"expected (...) or NAME or STRING, got {self.type}/{self.value}"
             )
-            raise AssertionError
 
     def expect(self, type: int, value: Optional[Any] = None) -> str:
         if self.type != type or (value is not None and self.value != value):
-            self.raise_error(
-                "expected %s/%s, got %s/%s", type, value, self.type, self.value
-            )
+            self.raise_error(f"expected {type}/{value}, got {self.type}/{self.value}")
         value = self.value
         self.gettoken()
         return value
@@ -351,12 +347,7 @@ def gettoken(self) -> None:
         self.type, self.value, self.begin, self.end, self.line = tup
         # print token.tok_name[self.type], repr(self.value)
 
-    def raise_error(self, msg: str, *args: Any) -> NoReturn:
-        if args:
-            try:
-                msg = msg % args
-            except Exception:
-                msg = " ".join([msg] + list(map(str, args)))
+    def raise_error(self, msg: str) -> NoReturn:
         raise SyntaxError(
             msg, (str(self.filename), self.end[0], self.end[1], self.line)
         )

src/blib2to3/pytree.py

@@ -268,11 +268,7 @@ def __init__(
     def __repr__(self) -> str:
         """Return a canonical string representation."""
         assert self.type is not None
-        return "{}({}, {!r})".format(
-            self.__class__.__name__,
-            type_repr(self.type),
-            self.children,
-        )
+        return f"{self.__class__.__name__}({type_repr(self.type)}, {self.children!r})"
 
     def __str__(self) -> str:
         """
@@ -421,10 +417,9 @@ def __repr__(self) -> str:
         from .pgen2.token import tok_name
 
         assert self.type is not None
-        return "{}({}, {!r})".format(
-            self.__class__.__name__,
-            tok_name.get(self.type, self.type),
-            self.value,
+        return (
+            f"{self.__class__.__name__}({tok_name.get(self.type, self.type)},"
+            f" {self.value!r})"
         )
 
     def __str__(self) -> str:
@@ -527,7 +522,7 @@ def __repr__(self) -> str:
         args = [type_repr(self.type), self.content, self.name]
         while args and args[-1] is None:
             del args[-1]
-        return "{}({})".format(self.__class__.__name__, ", ".join(map(repr, args)))
+        return f"{self.__class__.__name__}({', '.join(map(repr, args))})"
 
     def _submatch(self, node, results=None) -> bool:
         raise NotImplementedError

tests/data/cases/cantfit.py

@@ -31,7 +31,8 @@
         raise ValueError(err.format(key))
 concatenated_strings = "some strings that are " "concatenated implicitly, so if you put them on separate " "lines it will fit"
 del concatenated_strings, string_variable_name, normal_function_name, normal_name, need_more_to_make_the_line_long_enough
-
+del ([], name_1, name_2), [(), [], name_4, name_3], name_1[[name_2 for name_1 in name_0]]
+del (),
 
 # output
 
@@ -91,3 +92,9 @@
     normal_name,
     need_more_to_make_the_line_long_enough,
 )
+del (
+    ([], name_1, name_2),
+    [(), [], name_4, name_3],
+    name_1[[name_2 for name_1 in name_0]],
+)
+del ((),)

tests/data/cases/context_managers_39.py

@@ -84,6 +84,31 @@ async def func():
         pass
 
 
+
+# don't remove the brackets here, it changes the meaning of the code.
+with (x, y) as z:
+    pass
+
+
+# don't remove the brackets here, it changes the meaning of the code.
+# even though the code will always trigger a runtime error
+with (name_5, name_4), name_5:
+    pass
+
+
+def test_tuple_as_contextmanager():
+    from contextlib import nullcontext
+
+    try:
+        with (nullcontext(),nullcontext()),nullcontext():
+            pass
+    except TypeError: 
+        # test passed
+        pass
+    else:
+        # this should be a type error
+        assert False
+
 # output
 
 
@@ -172,3 +197,28 @@ async def func():
         some_other_function(argument1, argument2, argument3="some_value"),
     ):
         pass
+
+
+# don't remove the brackets here, it changes the meaning of the code.
+with (x, y) as z:
+    pass
+
+
+# don't remove the brackets here, it changes the meaning of the code.
+# even though the code will always trigger a runtime error
+with (name_5, name_4), name_5:
+    pass
+
+
+def test_tuple_as_contextmanager():
+    from contextlib import nullcontext
+
+    try:
+        with (nullcontext(), nullcontext()), nullcontext():
+            pass
+    except TypeError:
+        # test passed
+        pass
+    else:
+        # this should be a type error
+        assert False

tests/data/cases/fmtskip10.py

@@ -0,0 +1,9 @@
+# flags: --preview
+def foo(): return "mock"  # fmt: skip
+if True: print("yay")  # fmt: skip
+for i in range(10): print(i)  # fmt: skip
+
+j =     1 # fmt: skip
+while j < 10: j += 1  # fmt: skip
+
+b = [c for c in "A very long string that would normally generate some kind of collapse, since it is this long"] # fmt: skip

tests/data/cases/fmtskip11.py

@@ -0,0 +1,6 @@
+def foo():
+    pass
+
+
+# comment 1  # fmt: skip
+# comment 2

tests/data/cases/fstring_quotations.py

@@ -0,0 +1,67 @@
+# Regression tests for long f-strings, including examples from issue #3623
+
+a = (
+    'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+    f'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"{"b"}"'
+)
+
+a = (
+    f'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"{"b"}"'
+    'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+)
+
+a = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + \
+    f'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"{"b"}"'
+
+a = f'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"{"b"}"' + \
+    f'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"{"b"}"'
+
+a = (
+    f'bbbbbbb"{"b"}"'
+    'aaaaaaaa'
+)
+
+a = (
+    f'"{"b"}"'
+)
+
+a = (
+    f'\"{"b"}\"'
+)
+
+a = (
+    r'\"{"b"}\"'
+)
+
+# output
+
+# Regression tests for long f-strings, including examples from issue #3623
+
+a = (
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+    f'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"{"b"}"'
+)
+
+a = (
+    f'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"{"b"}"'
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+)
+
+a = (
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+    + f'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"{"b"}"'
+)
+
+a = (
+    f'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"{"b"}"'
+    + f'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"{"b"}"'
+)
+
+a = f'bbbbbbb"{"b"}"' "aaaaaaaa"
+
+a = f'"{"b"}"'
+
+a = f'"{"b"}"'
+
+a = r'\"{"b"}\"'
+

tests/data/cases/pep_572_py310.py

@@ -14,3 +14,8 @@
 f((a := b + c for c in range(10)), x)
 f(y=(a := b + c for c in range(10)))
 f(x, (a := b + c for c in range(10)), y=z, **q)
+
+
+# Don't remove parens when assignment expr is one of the exprs in a with statement
+with x, (a := b):
+    pass

tests/data/cases/python37.py

@@ -10,6 +10,7 @@ def g():
 
 
 async def func():
+    await ...
     if test:
         out_batched = [
             i
@@ -42,6 +43,7 @@ def g():
 
 
 async def func():
+    await ...
     if test:
         out_batched = [
             i

tests/data/cases/type_param_defaults.py

@@ -20,6 +20,8 @@ def trailing_comma1[T=int,](a: str):
 def trailing_comma2[T=int](a: str,):
     pass
 
+def weird_syntax[T=lambda: 42, **P=lambda: 43, *Ts=lambda: 44](): pass
+
 # output
 
 type A[T = int] = float
@@ -61,3 +63,7 @@ def trailing_comma2[T = int](
     a: str,
 ):
     pass
+
+
+def weird_syntax[T = lambda: 42, **P = lambda: 43, *Ts = lambda: 44]():
+    pass

tests/data/cases/type_params.py

@@ -13,6 +13,8 @@ def it_gets_worse[WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplit
 
 def magic[Trailing, Comma,](): pass
 
+def weird_syntax[T: lambda: 42, U: a or b](): pass
+
 # output
 
 
@@ -56,3 +58,7 @@ def magic[
     Comma,
 ]():
     pass
+
+
+def weird_syntax[T: lambda: 42, U: a or b]():
+    pass

tests/data/miscellaneous/debug_visitor.out

@@ -232,8 +232,6 @@ file_input
                         fstring
                           FSTRING_START
  "f'"
-                          FSTRING_MIDDLE
- ''
                           fstring_replacement_field
                             LBRACE
  '{'
@@ -242,8 +240,6 @@ file_input
                             RBRACE
  '}'
                           /fstring_replacement_field
-                          FSTRING_MIDDLE
- ''
                           fstring_replacement_field
                             LBRACE
  '{'
@@ -252,8 +248,6 @@ file_input
                             RBRACE
  '}'
                           /fstring_replacement_field
-                          FSTRING_MIDDLE
- ''
                           FSTRING_END
  "'"
                         /fstring
@@ -399,8 +393,6 @@ file_input
                         fstring
                           FSTRING_START
  "f'"
-                          FSTRING_MIDDLE
- ''
                           fstring_replacement_field
                             LBRACE
  '{'
@@ -419,8 +411,6 @@ file_input
                             RBRACE
  '}'
                           /fstring_replacement_field
-                          FSTRING_MIDDLE
- ''
                           FSTRING_END
  "'"
                         /fstring
@@ -549,8 +539,6 @@ file_input
                         fstring
                           FSTRING_START
  "f'"
-                          FSTRING_MIDDLE
- ''
                           fstring_replacement_field
                             LBRACE
  '{'
@@ -559,8 +547,6 @@ file_input
                             RBRACE
  '}'
                           /fstring_replacement_field
-                          FSTRING_MIDDLE
- ''
                           fstring_replacement_field
                             LBRACE
  '{'
@@ -569,8 +555,6 @@ file_input
                             RBRACE
  '}'
                           /fstring_replacement_field
-                          FSTRING_MIDDLE
- ''
                           FSTRING_END
  "'"
                         /fstring
@@ -660,8 +644,6 @@ file_input
                                 RBRACE
  '}'
                               /fstring_replacement_field
-                              FSTRING_MIDDLE
- ''
                               FSTRING_END
  "'"
                             /fstring
@@ -744,8 +726,6 @@ file_input
                             RBRACE
  '}'
                           /fstring_replacement_field
-                          FSTRING_MIDDLE
- ''
                           FSTRING_END
  "'"
                         /fstring

tests/test_black.py

@@ -14,6 +14,7 @@
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager, redirect_stderr
 from dataclasses import fields, replace
+from importlib.metadata import version as imp_version
 from io import BytesIO
 from pathlib import Path, WindowsPath
 from platform import system
@@ -25,6 +26,7 @@
 import pytest
 from click import unstyle
 from click.testing import CliRunner
+from packaging.version import Version
 from pathspec import PathSpec
 
 import black
@@ -114,7 +116,10 @@ class BlackRunner(CliRunner):
     """Make sure STDOUT and STDERR are kept separate when testing Black via its CLI."""
 
     def __init__(self) -> None:
-        super().__init__(mix_stderr=False)
+        if Version(imp_version("click")) >= Version("8.2.0"):
+            super().__init__()
+        else:
+            super().__init__(mix_stderr=False)  # type: ignore
 
 
 def invokeBlack(
@@ -187,10 +192,10 @@ def test_piping(self) -> None:
             input=BytesIO(source.encode("utf-8")),
         )
         self.assertEqual(result.exit_code, 0)
-        self.assertFormatEqual(expected, result.output)
-        if source != result.output:
-            black.assert_equivalent(source, result.output)
-            black.assert_stable(source, result.output, DEFAULT_MODE)
+        self.assertFormatEqual(expected, result.stdout)
+        if source != result.stdout:
+            black.assert_equivalent(source, result.stdout)
+            black.assert_stable(source, result.stdout, DEFAULT_MODE)
 
     def test_piping_diff(self) -> None:
         diff_header = re.compile(
@@ -210,7 +215,7 @@ def test_piping_diff(self) -> None:
             black.main, args, input=BytesIO(source.encode("utf-8"))
         )
         self.assertEqual(result.exit_code, 0)
-        actual = diff_header.sub(DETERMINISTIC_HEADER, result.output)
+        actual = diff_header.sub(DETERMINISTIC_HEADER, result.stdout)
         actual = actual.rstrip() + "\n"  # the diff output has a trailing space
         self.assertEqual(expected, actual)
 
@@ -295,7 +300,7 @@ def test_expression_diff(self) -> None:
             self.assertEqual(result.exit_code, 0)
         finally:
             os.unlink(tmp_file)
-        actual = result.output
+        actual = result.stdout
         actual = diff_header.sub(DETERMINISTIC_HEADER, actual)
         if expected != actual:
             dump = black.dump_to_file(actual)
@@ -404,7 +409,7 @@ def test_skip_magic_trailing_comma(self) -> None:
             self.assertEqual(result.exit_code, 0)
         finally:
             os.unlink(tmp_file)
-        actual = result.output
+        actual = result.stdout
         actual = diff_header.sub(DETERMINISTIC_HEADER, actual)
         actual = actual.rstrip() + "\n"  # the diff output has a trailing space
         if expected != actual:
@@ -417,21 +422,6 @@ def test_skip_magic_trailing_comma(self) -> None:
             )
             self.assertEqual(expected, actual, msg)
 
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_async_as_identifier(self) -> None:
-        source_path = get_case_path("miscellaneous", "async_as_identifier")
-        _, source, expected = read_data_from_file(source_path)
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        major, minor = sys.version_info[:2]
-        if major < 3 or (major <= 3 and minor < 7):
-            black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-        # ensure black can parse this when the target is 3.6
-        self.invokeBlack([str(source_path), "--target-version", "py36"])
-        # but not on 3.7, because async/await is no longer an identifier
-        self.invokeBlack([str(source_path), "--target-version", "py37"], exit_code=123)
-
     @patch("black.dump_to_file", dump_to_stderr)
     def test_python37(self) -> None:
         source_path = get_case_path("cases", "python37")
@@ -444,8 +434,6 @@ def test_python37(self) -> None:
         black.assert_stable(source, actual, DEFAULT_MODE)
         # ensure black can parse this when the target is 3.7
         self.invokeBlack([str(source_path), "--target-version", "py37"])
-        # but not on 3.6, because we use async as a reserved keyword
-        self.invokeBlack([str(source_path), "--target-version", "py36"], exit_code=123)
 
     def test_tab_comment_indentation(self) -> None:
         contents_tab = "if 1:\n\tif 2:\n\t\tpass\n\t# comment\n\tpass\n"
@@ -458,17 +446,6 @@ def test_tab_comment_indentation(self) -> None:
         self.assertFormatEqual(contents_spc, fs(contents_spc))
         self.assertFormatEqual(contents_spc, fs(contents_tab))
 
-        # mixed tabs and spaces (valid Python 2 code)
-        contents_tab = "if 1:\n        if 2:\n\t\tpass\n\t# comment\n        pass\n"
-        contents_spc = "if 1:\n    if 2:\n        pass\n    # comment\n    pass\n"
-        self.assertFormatEqual(contents_spc, fs(contents_spc))
-        self.assertFormatEqual(contents_spc, fs(contents_tab))
-
-        contents_tab = "if 1:\n        if 2:\n\t\tpass\n\t\t# comment\n        pass\n"
-        contents_spc = "if 1:\n    if 2:\n        pass\n        # comment\n    pass\n"
-        self.assertFormatEqual(contents_spc, fs(contents_spc))
-        self.assertFormatEqual(contents_spc, fs(contents_tab))
-
     def test_false_positive_symlink_output_issue_3384(self) -> None:
         # Emulate the behavior when using the CLI (`black ./child  --verbose`), which
         # involves patching some `pathlib.Path` methods. In particular, `is_dir` is
@@ -1826,7 +1803,7 @@ def test_bpo_2142_workaround(self) -> None:
             self.assertEqual(result.exit_code, 0)
         finally:
             os.unlink(tmp_file)
-        actual = result.output
+        actual = result.stdout
         actual = diff_header.sub(DETERMINISTIC_HEADER, actual)
         self.assertEqual(actual, expected)
 
@@ -1836,7 +1813,7 @@ def compare_results(
     ) -> None:
         """Helper method to test the value and exit code of a click Result."""
         assert (
-            result.output == expected_value
+            result.stdout == expected_value
         ), "The output did not match the expected value."
         assert result.exit_code == expected_exit_code, "The exit code is incorrect."
 
@@ -1913,7 +1890,8 @@ def test_code_option_safe(self) -> None:
             args = ["--safe", "--code", code]
             result = CliRunner().invoke(black.main, args)
 
-            self.compare_results(result, error_msg, 123)
+            assert error_msg == result.output
+            assert result.exit_code == 123
 
     def test_code_option_fast(self) -> None:
         """Test that the code option ignores errors when the sanity checks fail."""
@@ -1975,7 +1953,7 @@ def test_for_handled_unexpected_eof_error(self) -> None:
         with pytest.raises(black.parsing.InvalidInput) as exc_info:
             black.lib2to3_parse("print(", {})
 
-        exc_info.match("Cannot parse: 2:0: EOF in multi-line statement")
+        exc_info.match("Cannot parse: 1:6: Unexpected EOF in multi-line statement")
 
     def test_line_ranges_with_code_option(self) -> None:
         code = textwrap.dedent("""\
@@ -2070,6 +2048,26 @@ def test_lines_with_leading_tabs_expanded(self) -> None:
         assert lines_with_leading_tabs_expanded("\t\tx") == [f"{tab}{tab}x"]
         assert lines_with_leading_tabs_expanded("\tx\n  y") == [f"{tab}x", "  y"]
 
+    def test_backslash_carriage_return(self) -> None:
+        # These tests are here instead of in the normal cases because
+        # of git's newline normalization and because it's hard to
+        # get `\r` vs `\r\n` vs `\n` to display properly in editors
+        assert black.format_str("x=\\\r\n1", mode=black.FileMode()) == "x = 1\n"
+        assert black.format_str("x=\\\n1", mode=black.FileMode()) == "x = 1\n"
+        assert black.format_str("x=\\\r1", mode=black.FileMode()) == "x = 1\n"
+        assert (
+            black.format_str("class A\\\r\n:...", mode=black.FileMode())
+            == "class A: ...\n"
+        )
+        assert (
+            black.format_str("class A\\\n:...", mode=black.FileMode())
+            == "class A: ...\n"
+        )
+        assert (
+            black.format_str("class A\\\r:...", mode=black.FileMode())
+            == "class A: ...\n"
+        )
+
 
 class TestCaching:
     def test_get_cache_dir(

tests/test_tokenize.py

@@ -1,6 +1,5 @@
 """Tests for the blib2to3 tokenizer."""
 
-import io
 import sys
 import textwrap
 from dataclasses import dataclass
@@ -19,16 +18,10 @@ class Token:
 
 def get_tokens(text: str) -> list[Token]:
     """Return the tokens produced by the tokenizer."""
-    readline = io.StringIO(text).readline
-    tokens: list[Token] = []
-
-    def tokeneater(
-        type: int, string: str, start: tokenize.Coord, end: tokenize.Coord, line: str
-    ) -> None:
-        tokens.append(Token(token.tok_name[type], string, start, end))
-
-    tokenize.tokenize(readline, tokeneater)
-    return tokens
+    return [
+        Token(token.tok_name[tok_type], string, start, end)
+        for tok_type, string, start, end, _ in tokenize.tokenize(text)
+    ]
 
 
 def assert_tokenizes(text: str, tokens: list[Token]) -> None:
@@ -69,11 +62,9 @@ def test_fstring() -> None:
         'f"{x}"',
         [
             Token("FSTRING_START", 'f"', (1, 0), (1, 2)),
-            Token("FSTRING_MIDDLE", "", (1, 2), (1, 2)),
-            Token("LBRACE", "{", (1, 2), (1, 3)),
+            Token("OP", "{", (1, 2), (1, 3)),
             Token("NAME", "x", (1, 3), (1, 4)),
-            Token("RBRACE", "}", (1, 4), (1, 5)),
-            Token("FSTRING_MIDDLE", "", (1, 5), (1, 5)),
+            Token("OP", "}", (1, 4), (1, 5)),
             Token("FSTRING_END", '"', (1, 5), (1, 6)),
             Token("ENDMARKER", "", (2, 0), (2, 0)),
         ],
@@ -82,13 +73,11 @@ def test_fstring() -> None:
         'f"{x:y}"\n',
         [
             Token(type="FSTRING_START", string='f"', start=(1, 0), end=(1, 2)),
-            Token(type="FSTRING_MIDDLE", string="", start=(1, 2), end=(1, 2)),
-            Token(type="LBRACE", string="{", start=(1, 2), end=(1, 3)),
+            Token(type="OP", string="{", start=(1, 2), end=(1, 3)),
             Token(type="NAME", string="x", start=(1, 3), end=(1, 4)),
             Token(type="OP", string=":", start=(1, 4), end=(1, 5)),
             Token(type="FSTRING_MIDDLE", string="y", start=(1, 5), end=(1, 6)),
-            Token(type="RBRACE", string="}", start=(1, 6), end=(1, 7)),
-            Token(type="FSTRING_MIDDLE", string="", start=(1, 7), end=(1, 7)),
+            Token(type="OP", string="}", start=(1, 6), end=(1, 7)),
             Token(type="FSTRING_END", string='"', start=(1, 7), end=(1, 8)),
             Token(type="NEWLINE", string="\n", start=(1, 8), end=(1, 9)),
             Token(type="ENDMARKER", string="", start=(2, 0), end=(2, 0)),
@@ -99,10 +88,9 @@ def test_fstring() -> None:
         [
             Token(type="FSTRING_START", string='f"', start=(1, 0), end=(1, 2)),
             Token(type="FSTRING_MIDDLE", string="x\\\n", start=(1, 2), end=(2, 0)),
-            Token(type="LBRACE", string="{", start=(2, 0), end=(2, 1)),
+            Token(type="OP", string="{", start=(2, 0), end=(2, 1)),
             Token(type="NAME", string="a", start=(2, 1), end=(2, 2)),
-            Token(type="RBRACE", string="}", start=(2, 2), end=(2, 3)),
-            Token(type="FSTRING_MIDDLE", string="", start=(2, 3), end=(2, 3)),
+            Token(type="OP", string="}", start=(2, 2), end=(2, 3)),
             Token(type="FSTRING_END", string='"', start=(2, 3), end=(2, 4)),
             Token(type="NEWLINE", string="\n", start=(2, 4), end=(2, 5)),
             Token(type="ENDMARKER", string="", start=(3, 0), end=(3, 0)),

tox.ini

@@ -13,18 +13,16 @@ skip_install = True
 recreate = True
 deps =
     -r{toxinidir}/test_requirements.txt
-; parallelization is disabled on CI because pytest-dev/pytest-xdist#620 occurs too frequently
-; local runs can stay parallelized since they aren't rolling the dice so many times as like on CI
 commands =
     pip install -e .[d]
     coverage erase
     pytest tests --run-optional no_jupyter \
-        !ci: --numprocesses auto \
+        --numprocesses auto \
         --cov {posargs}
     pip install -e .[jupyter]
     pytest tests --run-optional jupyter \
         -m jupyter \
-        !ci: --numprocesses auto \
+        --numprocesses auto \
         --cov --cov-append {posargs}
     coverage report
 
@@ -34,20 +32,15 @@ skip_install = True
 recreate = True
 deps =
     -r{toxinidir}/test_requirements.txt
-; a separate worker is required in ci due to https://foss.heptapod.net/pypy/pypy/-/issues/3317
-; this seems to cause tox to wait forever
-; remove this when pypy releases the bugfix
 commands =
     pip install -e .[d]
     pytest tests \
         --run-optional no_jupyter \
-        !ci: --numprocesses auto \
-        ci: --numprocesses 1
+        --numprocesses auto
     pip install -e .[jupyter]
     pytest tests --run-optional jupyter \
         -m jupyter \
-        !ci: --numprocesses auto \
-        ci: --numprocesses 1
+        --numprocesses auto
 
 [testenv:{,ci-}311]
 setenv =
@@ -59,22 +52,17 @@ deps =
 ; We currently need > aiohttp 3.8.1 that is on PyPI for 3.11
     git+https://github.com/aio-libs/aiohttp
     -r{toxinidir}/test_requirements.txt
-; a separate worker is required in ci due to https://foss.heptapod.net/pypy/pypy/-/issues/3317
-; this seems to cause tox to wait forever
-; remove this when pypy releases the bugfix
 commands =
     pip install -e .[d]
     coverage erase
     pytest tests \
         --run-optional no_jupyter \
-        !ci: --numprocesses auto \
-        ci: --numprocesses 1 \
+        --numprocesses auto \
         --cov {posargs}
     pip install -e .[jupyter]
     pytest tests --run-optional jupyter \
         -m jupyter \
-        !ci: --numprocesses auto \
-        ci: --numprocesses 1 \
+        --numprocesses auto \
         --cov --cov-append {posargs}
     coverage report