Improve Python 2 only syntax detection (GH-2592)

* Improve Python 2 only syntax detection

First of all this fixes a mistake I made in Python 2 deprecation PR
using token.* to check for print/exec statements. Turns out that
for nodes with a type value higher than 256 its numeric type isn't
guaranteed to be constant. Using syms.* instead fixes this.

Also add support for the following cases:

    print "hello, world!"

    exec "print('hello, world!')"

    def set_position((x, y), value):
        pass

    try:
        pass
    except Exception, err:
        pass

    raise RuntimeError, "I feel like crashing today :p"

    `wow_these_really_did_exist`

    10L

* Add octal support, more test cases, and fixup long ints

Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>

Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>

CHANGES.md

@@ -1,5 +1,12 @@
 # Change Log
 
+## _Unreleased_
+
+### _Black_
+
+- Warn about Python 2 deprecation in more cases by improving Python 2 only syntax
+  detection (#2592)
+
 ## 21.10b0
 
 ### _Black_

src/black/__init__.py

@@ -1132,8 +1132,17 @@ def get_features_used(node: Node) -> Set[Feature]:  # noqa: C901
                 features.add(Feature.F_STRINGS)
 
         elif n.type == token.NUMBER:
-            if "_" in n.value:  # type: ignore
+            assert isinstance(n, Leaf)
+            if "_" in n.value:
                 features.add(Feature.NUMERIC_UNDERSCORES)
+            elif n.value.endswith(("L", "l")):
+                # Python 2: 10L
+                features.add(Feature.LONG_INT_LITERAL)
+            elif len(n.value) >= 2 and n.value[0] == "0" and n.value[1].isdigit():
+                # Python 2: 0123; 00123; ...
+                if not all(char == "0" for char in n.value):
+                    # although we don't want to match 0000 or similar
+                    features.add(Feature.OCTAL_INT_LITERAL)
 
         elif n.type == token.SLASH:
             if n.parent and n.parent.type in {
@@ -1171,10 +1180,31 @@ def get_features_used(node: Node) -> Set[Feature]:  # noqa: C901
                         if argch.type in STARS:
                             features.add(feature)
 
-        elif n.type == token.PRINT_STMT:
+        # Python 2 only features (for its deprecation) except for integers, see above
+        elif n.type == syms.print_stmt:
             features.add(Feature.PRINT_STMT)
-        elif n.type == token.EXEC_STMT:
+        elif n.type == syms.exec_stmt:
             features.add(Feature.EXEC_STMT)
+        elif n.type == syms.tfpdef:
+            # def set_position((x, y), value):
+            #     ...
+            features.add(Feature.AUTOMATIC_PARAMETER_UNPACKING)
+        elif n.type == syms.except_clause:
+            # try:
+            #     ...
+            # except Exception, err:
+            #     ...
+            if len(n.children) >= 4:
+                if n.children[-2].type == token.COMMA:
+                    features.add(Feature.COMMA_STYLE_EXCEPT)
+        elif n.type == syms.raise_stmt:
+            # raise Exception, "msg"
+            if len(n.children) >= 4:
+                if n.children[-2].type == token.COMMA:
+                    features.add(Feature.COMMA_STYLE_RAISE)
+        elif n.type == token.BACKQUOTE:
+            # `i'm surprised this ever existed`
+            features.add(Feature.BACKQUOTE_REPR)
 
     return features
 

src/black/mode.py

@@ -44,6 +44,12 @@ class Feature(Enum):
     # temporary for Python 2 deprecation
     PRINT_STMT = 200
     EXEC_STMT = 201
+    AUTOMATIC_PARAMETER_UNPACKING = 202
+    COMMA_STYLE_EXCEPT = 203
+    COMMA_STYLE_RAISE = 204
+    LONG_INT_LITERAL = 205
+    OCTAL_INT_LITERAL = 206
+    BACKQUOTE_REPR = 207
 
 
 VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = {
@@ -51,6 +57,12 @@ class Feature(Enum):
         Feature.ASYNC_IDENTIFIERS,
         Feature.PRINT_STMT,
         Feature.EXEC_STMT,
+        Feature.AUTOMATIC_PARAMETER_UNPACKING,
+        Feature.COMMA_STYLE_EXCEPT,
+        Feature.COMMA_STYLE_RAISE,
+        Feature.LONG_INT_LITERAL,
+        Feature.OCTAL_INT_LITERAL,
+        Feature.BACKQUOTE_REPR,
     },
     TargetVersion.PY33: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS},
     TargetVersion.PY34: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS},

src/blib2to3/pgen2/token.py

@@ -74,9 +74,6 @@
 COLONEQUAL: Final = 59
 N_TOKENS: Final = 60
 NT_OFFSET: Final = 256
-# temporary for Python 2 deprecation
-PRINT_STMT: Final = 316
-EXEC_STMT: Final = 288
 # --end constants--
 
 tok_name: Final[Dict[int, str]] = {}

tests/data/python2_detection.py

@@ -0,0 +1,90 @@
+# This uses a similar construction to the decorators.py test data file FYI.
+
+print "hello, world!"
+
+###
+
+exec "print('hello, world!')"
+
+###
+
+def set_position((x, y), value):
+    pass
+
+###
+
+try:
+    pass
+except Exception, err:
+    pass
+
+###
+
+raise RuntimeError, "I feel like crashing today :p"
+
+###
+
+`wow_these_really_did_exist`
+
+###
+
+10L
+
+###
+
+10l
+
+###
+
+0123
+
+# output
+
+print("hello python three!")
+
+###
+
+exec("I'm not sure if you can use exec like this but that's not important here!")
+
+###
+
+try:
+    pass
+except make_exception(1, 2):
+    pass
+
+###
+
+try:
+    pass
+except Exception as err:
+    pass
+
+###
+
+raise RuntimeError(make_msg(1, 2))
+
+###
+
+raise RuntimeError("boom!",)
+
+###
+
+def set_position(x, y, value):
+    pass
+
+###
+
+10
+
+###
+
+0
+
+###
+
+000
+
+###
+
+0o12

tests/test_black.py

@@ -2017,6 +2017,7 @@ def test_get_sources_with_stdin_filename_and_force_exclude(self) -> None:
         )
 
 
+@pytest.mark.python2
 @pytest.mark.parametrize("explicit", [True, False], ids=["explicit", "autodetection"])
 def test_python_2_deprecation_with_target_version(explicit: bool) -> None:
     args = [
@@ -2032,6 +2033,20 @@ def test_python_2_deprecation_with_target_version(explicit: bool) -> None:
     assert "DEPRECATION: Python 2 support will be removed" in result.stderr
 
 
+@pytest.mark.python2
+def test_python_2_deprecation_autodetection_extended() -> None:
+    # this test has a similar construction to test_get_features_used_decorator
+    python2, non_python2 = read_data("python2_detection")
+    for python2_case in python2.split("###"):
+        node = black.lib2to3_parse(python2_case)
+        assert black.detect_target_versions(node) == {TargetVersion.PY27}, python2_case
+    for non_python2_case in non_python2.split("###"):
+        node = black.lib2to3_parse(non_python2_case)
+        assert black.detect_target_versions(node) != {
+            TargetVersion.PY27
+        }, non_python2_case
+
+
 with open(black.__file__, "r", encoding="utf-8") as _bf:
     black_source_lines = _bf.readlines()