diff --git a/CHANGES.md b/CHANGES.md index ae8bb78..f5b123e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,9 @@ - Fix crash when a tuple appears in the `as` clause of a `with` statement (#4634) - Fix crash when tuple is used as a context manager inside a `with` statement (#4646) - Fix crash on a `\\r\n` (#4673) +- Fix crash on `await ...` (where `...` is a literal `Ellipsis`) (#4676) +- Remove support for pre-python 3.7 `await/async` as soft keywords/variable names + (#4676) ### Preview style diff --git a/docs/faq.md b/docs/faq.md index 51db1c9..9efcf31 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -93,6 +93,8 @@ Support for formatting Python 2 code was removed in version 22.0. While we've ma plans to stop supporting older Python 3 minor versions immediately, their support might also be removed some time in the future without a deprecation period. +`await`/`async` as soft keywords/indentifiers are no longer supported as of 25.2.0. + Runtime support for 3.6 was removed in version 22.10.0, for 3.7 in version 23.7.0, and for 3.8 in version 24.10.0. diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index 46c4319..1d51563 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -138,20 +138,13 @@ def transform_whitespace( def tokenize(source: str, grammar: Optional[Grammar] = None) -> Iterator[TokenInfo]: - async_keywords = False if grammar is None else grammar.async_keywords - lines = source.split("\n") lines += [""] # For newline tokens in files that don't end in a newline line, column = 1, 0 - token_iterator = pytokens.tokenize(source) - is_async = False - current_indent = 0 - async_indent = 0 - prev_token: Optional[pytokens.Token] = None try: - for token in token_iterator: + for token in pytokens.tokenize(source): token = transform_whitespace(token, source, prev_token) line, column = token.start_line, token.start_col @@ -166,58 +159,18 @@ def tokenize(source: str, grammar: Optional[Grammar] = None) -> Iterator[TokenIn prev_token = token continue - if token.type == TokenType.indent: - current_indent += 1 - if token.type == TokenType.dedent: - current_indent -= 1 - if is_async and current_indent < async_indent: - is_async = False - source_line = lines[token.start_line - 1] if token.type == TokenType.identifier and token_str in ("async", "await"): # Black uses `async` and `await` token types just for those two keywords - while True: - next_token = next(token_iterator) - next_str = source[next_token.start_index : next_token.end_index] - next_token = transform_whitespace(next_token, next_str, token) - if next_token.type == TokenType.whitespace: - continue - break - - next_token_type = TOKEN_TYPE_MAP[next_token.type] - next_line = lines[next_token.start_line - 1] - - if token_str == "async" and ( - async_keywords - or (next_token_type == NAME and next_str in ("def", "for")) - ): - is_async = True - async_indent = current_indent + 1 - current_token_type = ASYNC - elif token_str == "await" and (async_keywords or is_async): - current_token_type = AWAIT - else: - current_token_type = TOKEN_TYPE_MAP[token.type] - yield ( - current_token_type, + ASYNC if token_str == "async" else AWAIT, token_str, (token.start_line, token.start_col), (token.end_line, token.end_col), source_line, ) - yield ( - next_token_type, - next_str, - (next_token.start_line, next_token.start_col), - (next_token.end_line, next_token.end_col), - next_line, - ) - prev_token = token - continue - - if token.type == TokenType.op and token_str == "...": + elif token.type == TokenType.op and token_str == "...": # Black doesn't have an ellipsis token yet, yield 3 DOTs instead assert token.start_line == token.end_line assert token.end_col == token.start_col + 3 @@ -232,16 +185,14 @@ def tokenize(source: str, grammar: Optional[Grammar] = None) -> Iterator[TokenIn (token.end_line, end_col), source_line, ) - prev_token = token - continue - - yield ( - TOKEN_TYPE_MAP[token.type], - token_str, - (token.start_line, token.start_col), - (token.end_line, token.end_col), - source_line, - ) + else: + yield ( + TOKEN_TYPE_MAP[token.type], + token_str, + (token.start_line, token.start_col), + (token.end_line, token.end_col), + source_line, + ) prev_token = token except pytokens.UnexpectedEOF: diff --git a/tests/data/cases/python37.py b/tests/data/cases/python37.py index f69f6b4..a600621 100644 --- a/tests/data/cases/python37.py +++ b/tests/data/cases/python37.py @@ -10,6 +10,7 @@ def g(): async def func(): + await ... if test: out_batched = [ i @@ -42,6 +43,7 @@ def g(): async def func(): + await ... if test: out_batched = [ i diff --git a/tests/test_black.py b/tests/test_black.py index 4588add..f0a5fc7 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -422,21 +422,6 @@ def test_skip_magic_trailing_comma(self) -> None: ) self.assertEqual(expected, actual, msg) - @patch("black.dump_to_file", dump_to_stderr) - def test_async_as_identifier(self) -> None: - source_path = get_case_path("miscellaneous", "async_as_identifier") - _, source, expected = read_data_from_file(source_path) - actual = fs(source) - self.assertFormatEqual(expected, actual) - major, minor = sys.version_info[:2] - if major < 3 or (major <= 3 and minor < 7): - black.assert_equivalent(source, actual) - black.assert_stable(source, actual, DEFAULT_MODE) - # ensure black can parse this when the target is 3.6 - self.invokeBlack([str(source_path), "--target-version", "py36"]) - # but not on 3.7, because async/await is no longer an identifier - self.invokeBlack([str(source_path), "--target-version", "py37"], exit_code=123) - @patch("black.dump_to_file", dump_to_stderr) def test_python37(self) -> None: source_path = get_case_path("cases", "python37") @@ -449,8 +434,6 @@ def test_python37(self) -> None: black.assert_stable(source, actual, DEFAULT_MODE) # ensure black can parse this when the target is 3.7 self.invokeBlack([str(source_path), "--target-version", "py37"]) - # but not on 3.6, because we use async as a reserved keyword - self.invokeBlack([str(source_path), "--target-version", "py36"], exit_code=123) def test_tab_comment_indentation(self) -> None: contents_tab = "if 1:\n\tif 2:\n\t\tpass\n\t# comment\n\tpass\n"