Add underscores to numeric literals with more than six digits (#529)

README.md

@@ -381,6 +381,11 @@ styled as `2L` instead of `2l` to avoid confusion between `l` and `1`. In
 Python 3.6+, *Black* adds underscores to long numeric literals to aid
 readability: `100000000` becomes `100_000_000`.
 
+For regions where numerals are grouped differently (like [India](https://en.wikipedia.org/wiki/Indian_numbering_system)
+and [China](https://en.wikipedia.org/wiki/Chinese_numerals#Whole_numbers)),
+the `-N` or `--skip-numeric-underscore-normalization` command line option
+makes *Black* preserve underscores in numeric literals.
+
 ### Line breaks & binary operators
 
 *Black* will break a line before a binary operator when splitting a block
@@ -796,6 +801,8 @@ The headers controlling how code is formatted are:
  - `X-Skip-String-Normalization`: corresponds to the `--skip-string-normalization`
     command line flag. If present and its value is not the empty string, no string
     normalization will be performed.
+ - `X-Skip-Numeric-Underscore-Normalization`: corresponds to the
+    `--skip-numeric-underscore-normalization` command line flag.
  - `X-Fast-Or-Safe`: if set to `fast`, `blackd` will act as *Black* does when
     passed the `--fast` command line flag.
  - `X-Python-Variant`: if set to `pyi`, `blackd` will act as *Black* does when
@@ -926,6 +933,9 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
 
   * numeric literals are normalized to include `_` separators on Python 3.6+ code
 
+  * added `--skip-numeric-underscore-normalization` to disable the above behavior and
+    leave numeric underscores as they were in the input
+
   * code with `_` in numeric literals is recognized as Python 3.6+
 
   * most letters in numeric literals are lowercased (e.g., in `1e10` or `0xab`)

black.py

@@ -115,10 +115,16 @@ class FileMode(Flag):
     PYTHON36 = 1
     PYI = 2
     NO_STRING_NORMALIZATION = 4
+    NO_NUMERIC_UNDERSCORE_NORMALIZATION = 8
 
     @classmethod
     def from_configuration(
-        cls, *, py36: bool, pyi: bool, skip_string_normalization: bool
+        cls,
+        *,
+        py36: bool,
+        pyi: bool,
+        skip_string_normalization: bool,
+        skip_numeric_underscore_normalization: bool,
     ) -> "FileMode":
         mode = cls.AUTO_DETECT
         if py36:
@@ -127,6 +133,8 @@ def from_configuration(
             mode |= cls.PYI
         if skip_string_normalization:
             mode |= cls.NO_STRING_NORMALIZATION
+        if skip_numeric_underscore_normalization:
+            mode |= cls.NO_NUMERIC_UNDERSCORE_NORMALIZATION
         return mode
 
 
@@ -196,6 +204,12 @@ def read_pyproject_toml(
     is_flag=True,
     help="Don't normalize string quotes or prefixes.",
 )
+@click.option(
+    "-N",
+    "--skip-numeric-underscore-normalization",
+    is_flag=True,
+    help="Don't normalize underscores in numeric literals.",
+)
 @click.option(
     "--check",
     is_flag=True,
@@ -286,6 +300,7 @@ def main(
     pyi: bool,
     py36: bool,
     skip_string_normalization: bool,
+    skip_numeric_underscore_normalization: bool,
     quiet: bool,
     verbose: bool,
     include: str,
@@ -296,7 +311,10 @@ def main(
     """The uncompromising code formatter."""
     write_back = WriteBack.from_configuration(check=check, diff=diff)
     mode = FileMode.from_configuration(
-        py36=py36, pyi=pyi, skip_string_normalization=skip_string_normalization
+        py36=py36,
+        pyi=pyi,
+        skip_string_normalization=skip_string_normalization,
+        skip_numeric_underscore_normalization=skip_numeric_underscore_normalization,
     )
     if config and verbose:
         out(f"Using configuration from {config}.", bold=False, fg="blue")
@@ -618,7 +636,8 @@ def format_str(
         remove_u_prefix=py36 or "unicode_literals" in future_imports,
         is_pyi=is_pyi,
         normalize_strings=normalize_strings,
-        allow_underscores=py36,
+        allow_underscores=py36
+        and not bool(mode & FileMode.NO_NUMERIC_UNDERSCORE_NORMALIZATION),
     )
     elt = EmptyLineTracker(is_pyi=is_pyi)
     empty_line = Line()
@@ -2600,8 +2619,8 @@ def format_int_string(
         return text
 
     text = text.replace("_", "")
-    if len(text) <= 6:
+    if len(text) <= 5:
-        # No underscores for numbers <= 6 digits long.
+        # No underscores for numbers <= 5 digits long.
         return text
 
     if count_from_end:

blackd.py

@@ -14,6 +14,7 @@
 LINE_LENGTH_HEADER = "X-Line-Length"
 PYTHON_VARIANT_HEADER = "X-Python-Variant"
 SKIP_STRING_NORMALIZATION_HEADER = "X-Skip-String-Normalization"
+SKIP_NUMERIC_UNDERSCORE_NORMALIZATION_HEADER = "X-Skip-Numeric-Underscore-Normalization"
 FAST_OR_SAFE_HEADER = "X-Fast-Or-Safe"
 
 
@@ -69,11 +70,17 @@ async def handle(request: web.Request, executor: Executor) -> web.Response:
         skip_string_normalization = bool(
             request.headers.get(SKIP_STRING_NORMALIZATION_HEADER, False)
         )
+        skip_numeric_underscore_normalization = bool(
+            request.headers.get(SKIP_NUMERIC_UNDERSCORE_NORMALIZATION_HEADER, False)
+        )
         fast = False
         if request.headers.get(FAST_OR_SAFE_HEADER, "safe") == "fast":
             fast = True
         mode = black.FileMode.from_configuration(
-            py36=py36, pyi=pyi, skip_string_normalization=skip_string_normalization
+            py36=py36,
+            pyi=pyi,
+            skip_string_normalization=skip_string_normalization,
+            skip_numeric_underscore_normalization=skip_numeric_underscore_normalization,
         )
         req_bytes = await request.content.read()
         charset = request.charset if request.charset is not None else "utf8"

tests/data/fmtonoff.py

@@ -225,7 +225,7 @@ def function_signature_stress_test(number:int,no_annotation=None,text:str='defau
  return text[number:-1]
 # fmt: on
 def spaces(a=1, b=(), c=[], d={}, e=True, f=-1, g=1 if False else 2, h="", i=r""):
-    offset = attr.ib(default=attr.Factory(lambda: _r.uniform(10000, 200000)))
+    offset = attr.ib(default=attr.Factory(lambda: _r.uniform(10000, 200_000)))
     assert task._cancel_stack[: len(old_stack)] == old_stack
 
 

tests/data/function.py

@@ -144,7 +144,7 @@ def function_signature_stress_test(
 
 
 def spaces(a=1, b=(), c=[], d={}, e=True, f=-1, g=1 if False else 2, h="", i=r""):
-    offset = attr.ib(default=attr.Factory(lambda: _r.uniform(10000, 200000)))
+    offset = attr.ib(default=attr.Factory(lambda: _r.uniform(10000, 200_000)))
     assert task._cancel_stack[: len(old_stack)] == old_stack
 
 

tests/data/numeric_literals.py

@@ -16,6 +16,8 @@
 x = 0B1011
 x = 0O777
 x = 0.000000006
+x = 10000
+x = 133333
 
 # output
 
@@ -23,7 +25,7 @@
 #!/usr/bin/env python3.6
 
 x = 123_456_789
-x = 123456
+x = 123_456
 x = 0.1
 x = 1.0
 x = 1e1
@@ -38,3 +40,5 @@
 x = 0b1011
 x = 0o777
 x = 0.000_000_006
+x = 10000
+x = 133_333

tests/data/numeric_literals_skip_underscores.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3.6
+
+x = 123456789
+x = 1_2_3_4_5_6_7
+x = 1E+1
+x = 0xb1acc
+x = 0.00_00_006
+x = 12_34_567J
+x = .1_2
+x = 1_2.
+
+# output
+
+#!/usr/bin/env python3.6
+
+x = 123456789
+x = 1_2_3_4_5_6_7
+x = 1e1
+x = 0xB1ACC
+x = 0.00_00_006
+x = 12_34_567j
+x = 0.1_2
+x = 1_2.0

tests/test_black.py

@@ -410,6 +410,17 @@ def test_numeric_literals(self) -> None:
         black.assert_equivalent(source, actual)
         black.assert_stable(source, actual, line_length=ll)
 
+    @patch("black.dump_to_file", dump_to_stderr)
+    def test_numeric_literals_ignoring_underscores(self) -> None:
+        source, expected = read_data("numeric_literals_skip_underscores")
+        mode = (
+            black.FileMode.PYTHON36 | black.FileMode.NO_NUMERIC_UNDERSCORE_NORMALIZATION
+        )
+        actual = fs(source, mode=mode)
+        self.assertFormatEqual(expected, actual)
+        black.assert_equivalent(source, actual)
+        black.assert_stable(source, actual, line_length=ll, mode=mode)
+
     @patch("black.dump_to_file", dump_to_stderr)
     def test_numeric_literals_py2(self) -> None:
         source, expected = read_data("numeric_literals_py2")