Fix await ellipses and remove async/await soft keyword/identifier support (#4676)

* Update tokenize.py

* Update driver.py

* Update test_black.py

* Update test_black.py

* Update python37.py

* Update tokenize.py

* Update CHANGES.md

* Update CHANGES.md

* Update faq.md

* Update driver.py

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

CHANGES.md

@@ -17,6 +17,9 @@
 - Fix crash when a tuple appears in the `as` clause of a `with` statement (#4634)
 - Fix crash when tuple is used as a context manager inside a `with` statement (#4646)
 - Fix crash on a `\\r\n` (#4673)
+- Fix crash on `await ...` (where `...` is a literal `Ellipsis`) (#4676)
+- Remove support for pre-python 3.7 `await/async` as soft keywords/variable names
+  (#4676)
 
 ### Preview style
 

docs/faq.md

@@ -93,6 +93,8 @@ Support for formatting Python 2 code was removed in version 22.0. While we've ma
 plans to stop supporting older Python 3 minor versions immediately, their support might
 also be removed some time in the future without a deprecation period.
 
+`await`/`async` as soft keywords/indentifiers are no longer supported as of 25.2.0.
+
 Runtime support for 3.6 was removed in version 22.10.0, for 3.7 in version 23.7.0, and
 for 3.8 in version 24.10.0.
 

src/blib2to3/pgen2/tokenize.py

@@ -138,20 +138,13 @@ def transform_whitespace(
 
 
 def tokenize(source: str, grammar: Optional[Grammar] = None) -> Iterator[TokenInfo]:
-    async_keywords = False if grammar is None else grammar.async_keywords
-
     lines = source.split("\n")
     lines += [""]  # For newline tokens in files that don't end in a newline
     line, column = 1, 0
 
-    token_iterator = pytokens.tokenize(source)
-    is_async = False
-    current_indent = 0
-    async_indent = 0
-
     prev_token: Optional[pytokens.Token] = None
     try:
-        for token in token_iterator:
+        for token in pytokens.tokenize(source):
             token = transform_whitespace(token, source, prev_token)
 
             line, column = token.start_line, token.start_col
@@ -166,58 +159,18 @@ def tokenize(source: str, grammar: Optional[Grammar] = None) -> Iterator[TokenIn
                 prev_token = token
                 continue
 
-            if token.type == TokenType.indent:
-                current_indent += 1
-            if token.type == TokenType.dedent:
-                current_indent -= 1
-                if is_async and current_indent < async_indent:
-                    is_async = False
-
             source_line = lines[token.start_line - 1]
 
             if token.type == TokenType.identifier and token_str in ("async", "await"):
                 # Black uses `async` and `await` token types just for those two keywords
-                while True:
-                    next_token = next(token_iterator)
-                    next_str = source[next_token.start_index : next_token.end_index]
-                    next_token = transform_whitespace(next_token, next_str, token)
-                    if next_token.type == TokenType.whitespace:
-                        continue
-                    break
-
-                next_token_type = TOKEN_TYPE_MAP[next_token.type]
-                next_line = lines[next_token.start_line - 1]
-
-                if token_str == "async" and (
-                    async_keywords
-                    or (next_token_type == NAME and next_str in ("def", "for"))
-                ):
-                    is_async = True
-                    async_indent = current_indent + 1
-                    current_token_type = ASYNC
-                elif token_str == "await" and (async_keywords or is_async):
-                    current_token_type = AWAIT
-                else:
-                    current_token_type = TOKEN_TYPE_MAP[token.type]
-
                 yield (
-                    current_token_type,
+                    ASYNC if token_str == "async" else AWAIT,
                     token_str,
                     (token.start_line, token.start_col),
                     (token.end_line, token.end_col),
                     source_line,
                 )
-                yield (
+            elif token.type == TokenType.op and token_str == "...":
-                    next_token_type,
-                    next_str,
-                    (next_token.start_line, next_token.start_col),
-                    (next_token.end_line, next_token.end_col),
-                    next_line,
-                )
-                prev_token = token
-                continue
-
-            if token.type == TokenType.op and token_str == "...":
                 # Black doesn't have an ellipsis token yet, yield 3 DOTs instead
                 assert token.start_line == token.end_line
                 assert token.end_col == token.start_col + 3
@@ -232,16 +185,14 @@ def tokenize(source: str, grammar: Optional[Grammar] = None) -> Iterator[TokenIn
                         (token.end_line, end_col),
                         source_line,
                     )
-                prev_token = token
+            else:
-                continue
+                yield (
-
+                    TOKEN_TYPE_MAP[token.type],
-            yield (
+                    token_str,
-                TOKEN_TYPE_MAP[token.type],
+                    (token.start_line, token.start_col),
-                token_str,
+                    (token.end_line, token.end_col),
-                (token.start_line, token.start_col),
+                    source_line,
-                (token.end_line, token.end_col),
+                )
-                source_line,
-            )
             prev_token = token
 
     except pytokens.UnexpectedEOF:

tests/data/cases/python37.py

@@ -10,6 +10,7 @@ def g():
 
 
 async def func():
+    await ...
     if test:
         out_batched = [
             i
@@ -42,6 +43,7 @@ def g():
 
 
 async def func():
+    await ...
     if test:
         out_batched = [
             i

tests/test_black.py

@@ -422,21 +422,6 @@ def test_skip_magic_trailing_comma(self) -> None:
             )
             self.assertEqual(expected, actual, msg)
 
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_async_as_identifier(self) -> None:
-        source_path = get_case_path("miscellaneous", "async_as_identifier")
-        _, source, expected = read_data_from_file(source_path)
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        major, minor = sys.version_info[:2]
-        if major < 3 or (major <= 3 and minor < 7):
-            black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-        # ensure black can parse this when the target is 3.6
-        self.invokeBlack([str(source_path), "--target-version", "py36"])
-        # but not on 3.7, because async/await is no longer an identifier
-        self.invokeBlack([str(source_path), "--target-version", "py37"], exit_code=123)
-
     @patch("black.dump_to_file", dump_to_stderr)
     def test_python37(self) -> None:
         source_path = get_case_path("cases", "python37")
@@ -449,8 +434,6 @@ def test_python37(self) -> None:
         black.assert_stable(source, actual, DEFAULT_MODE)
         # ensure black can parse this when the target is 3.7
         self.invokeBlack([str(source_path), "--target-version", "py37"])
-        # but not on 3.6, because we use async as a reserved keyword
-        self.invokeBlack([str(source_path), "--target-version", "py36"], exit_code=123)
 
     def test_tab_comment_indentation(self) -> None:
         contents_tab = "if 1:\n\tif 2:\n\t\tpass\n\t# comment\n\tpass\n"