Automatic detection of deprecated Python 2 forms of print and exec

Note: if those are handled, you can't use --safe because this check is using
Python 3.6+ builtin AST.

Fixes #49

README.md

@@ -275,8 +275,7 @@ python setup.py test
 
 But you can reformat Python 2 code with it, too.  *Black* is able to parse
 all of the new syntax supported on Python 3.6 but also *effectively all*
-the Python 2 syntax at the same time, as long as you're not using print
+the Python 2 syntax at the same time.
-statements.
 
 By making the code exclusively Python 3.6+, I'm able to focus on the
 quality of the formatting and re-use all the nice features of the new
@@ -309,6 +308,9 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
 
 ### 18.3a4 (unreleased)
 
+* automatic detection of deprecated Python 2 forms of print statements
+  and exec statements in the formatted file (#49)
+
 * only return exit code 1 when --check is used (#50)
 
 * don't remove single trailing commas from square bracket indexing

black.py

@@ -235,23 +235,36 @@ def format_str(src_contents: str, line_length: int) -> FileContent:
     return dst_contents
 
 
+GRAMMARS = [
+    pygram.python_grammar_no_print_statement_no_exec_statement,
+    pygram.python_grammar_no_print_statement,
+    pygram.python_grammar_no_exec_statement,
+    pygram.python_grammar,
+]
+
+
 def lib2to3_parse(src_txt: str) -> Node:
     """Given a string with source, return the lib2to3 Node."""
     grammar = pygram.python_grammar_no_print_statement
-    drv = driver.Driver(grammar, pytree.convert)
     if src_txt[-1] != '\n':
         nl = '\r\n' if '\r\n' in src_txt[:1024] else '\n'
         src_txt += nl
-    try:
+    for grammar in GRAMMARS:
-        result = drv.parse_string(src_txt, True)
+        drv = driver.Driver(grammar, pytree.convert)
-    except ParseError as pe:
-        lineno, column = pe.context[1]
-        lines = src_txt.splitlines()
         try:
-            faulty_line = lines[lineno - 1]
+            result = drv.parse_string(src_txt, True)
-        except IndexError:
+            break
-            faulty_line = "<line number missing in source>"
+
-        raise ValueError(f"Cannot parse: {lineno}:{column}: {faulty_line}") from None
+        except ParseError as pe:
+            lineno, column = pe.context[1]
+            lines = src_txt.splitlines()
+            try:
+                faulty_line = lines[lineno - 1]
+            except IndexError:
+                faulty_line = "<line number missing in source>"
+            exc = ValueError(f"Cannot parse: {lineno}:{column}: {faulty_line}")
+    else:
+        raise exc from None
 
     if isinstance(result, Leaf):
         result = Node(syms.file_input, [result])
@@ -903,6 +916,17 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
         ):
             return NO
 
+        elif (
+            prevp.type == token.RIGHTSHIFT
+            and prevp.parent
+            and prevp.parent.type == syms.shift_expr
+            and prevp.prev_sibling
+            and prevp.prev_sibling.type == token.NAME
+            and prevp.prev_sibling.value == 'print'
+        ):
+            # Python 2 print chevron
+            return NO
+
     elif prev.type in OPENING_BRACKETS:
         return NO
 
@@ -1538,7 +1562,12 @@ def _v(node: ast.AST, depth: int = 0) -> Iterator[str]:
     try:
         src_ast = ast.parse(src)
     except Exception as exc:
-        raise AssertionError(f"cannot parse source: {exc}") from None
+        major, minor = sys.version_info[:2]
+        raise AssertionError(
+            f"cannot use --safe with this file; failed to parse source file "
+            f"with Python {major}.{minor}'s builtin AST. Re-run with --fast "
+            f"or stop using deprecated Python 2 syntax. AST error message: {exc}"
+        )
 
     try:
         dst_ast = ast.parse(dst)

blib2to3/pygram.py

@@ -36,5 +36,12 @@ def __init__(self, grammar):
 python_grammar_no_print_statement = python_grammar.copy()
 del python_grammar_no_print_statement.keywords["print"]
 
+python_grammar_no_exec_statement = python_grammar.copy()
+del python_grammar_no_exec_statement.keywords["exec"]
+
+python_grammar_no_print_statement_no_exec_statement = python_grammar.copy()
+del python_grammar_no_print_statement_no_exec_statement.keywords["print"]
+del python_grammar_no_print_statement_no_exec_statement.keywords["exec"]
+
 pattern_grammar = driver.load_packaged_grammar("blib2to3", _PATTERN_GRAMMAR_FILE)
 pattern_symbols = Symbols(pattern_grammar)

blib2to3/pygram.pyi

@@ -116,4 +116,6 @@ class pattern_symbols(Symbols):
 
 python_grammar: Grammar
 python_grammar_no_print_statement: Grammar
+python_grammar_no_print_statement_no_exec_statement: Grammar
+python_grammar_no_exec_statement: Grammar
 pattern_grammar: Grammar

tests/function.py

@@ -14,8 +14,9 @@ def func_no_args():
   for i in range(10):
     print(i)
     continue
+  exec("new-style exec", {}, {})
   return None
-async def coroutine(arg):
+async def coroutine(arg, exec=False):
  "Single-line docstring. Multiline is harder to reformat."
  async with some_connection() as conn:
      await conn.do_what_i_mean('SELECT bobby, tables FROM xkcd', timeout=2)
@@ -93,10 +94,11 @@ def func_no_args():
         print(i)
         continue
 
+    exec("new-style exec", {}, {})
     return None
 
 
-async def coroutine(arg):
+async def coroutine(arg, exec=False):
     "Single-line docstring. Multiline is harder to reformat."
     async with some_connection() as conn:
         await conn.do_what_i_mean('SELECT bobby, tables FROM xkcd', timeout=2)

tests/python2.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python2
+
+import sys
+
+print >> sys.stderr , "Warning:" ,
+print >> sys.stderr , "this is a blast from the past."
+print >> sys.stderr , "Look, a repr:", `sys`
+
+
+def function((_globals, _locals)):
+    exec "print 'hi from exec!'" in _globals, _locals
+
+
+function((globals(), locals()))
+
+
+# output
+
+
+#!/usr/bin/env python2
+
+import sys
+
+print >>sys.stderr, "Warning:",
+print >>sys.stderr, "this is a blast from the past."
+print >>sys.stderr, "Look, a repr:", ` sys `
+
+
+def function((_globals, _locals)):
+    exec "print 'hi from exec!'" in _globals, _locals
+
+
+function((globals(), locals()))

tests/test_black.py

@@ -180,6 +180,14 @@ def test_empty_lines(self) -> None:
         black.assert_equivalent(source, actual)
         black.assert_stable(source, actual, line_length=ll)
 
+    @patch("black.dump_to_file", dump_to_stderr)
+    def test_python2(self) -> None:
+        source, expected = read_data('python2')
+        actual = fs(source)
+        self.assertFormatEqual(expected, actual)
+        # black.assert_equivalent(source, actual)
+        black.assert_stable(source, actual, line_length=ll)
+
     def test_report(self) -> None:
         report = black.Report()
         out_lines = []