actions/black
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for all valid string literals (#115)

Browse Source
This commit is contained in:
Zsolt Dollenstein 2018-04-09 22:36:40 +01:00 committed by Łukasz Langa
parent ecc294741e
commit ecdbf085a7
5 changed files with 34 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
black.py
View File

@ -1923,8 +1923,8 @@ def normalize_string_quotes(leaf: Leaf) -> None:
prefix = leaf.value[:first_quote_pos]
body = leaf.value[first_quote_pos + len(orig_quote):-len(orig_quote)]
unescaped_new_quote = re.compile(r"(([^\\]|^)(\\\\)*)" + new_quote)
escaped_orig_quote = re.compile(r"\\(\\\\)*" + orig_quote)
unescaped_new_quote = re.compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
escaped_orig_quote = re.compile(rf"\\(\\\\)*{orig_quote}")
if "r" in prefix.casefold():
if unescaped_new_quote.search(body):
# There's at least one unescaped new_quote in this raw string
@ -1934,8 +1934,8 @@ def normalize_string_quotes(leaf: Leaf) -> None:
# Do not introduce or remove backslashes in raw strings
new_body = body
else:
new_body = escaped_orig_quote.sub(f"\\1{orig_quote}", body)
new_body = unescaped_new_quote.sub(f"\\1\\\\{new_quote}", new_body)
new_body = escaped_orig_quote.sub(rf"\1{orig_quote}", body)
new_body = unescaped_new_quote.sub(rf"\1\\{new_quote}", new_body)
if new_quote == '"""' and new_body[-1] == '"':
# edge case:
new_body = new_body[:-1] + '\\"'

1
blib2to3/README
View File

@ -7,6 +7,7 @@ Reasons for forking:
*args and **kwargs
- backport of GH-6143 that restores the ability to reformat legacy usage of
`async`
- support all types of string literals
- better ability to debug (better reprs)
- INDENT and DEDENT don't hold whitespace and comment prefixes
- ability to Cythonize

76
blib2to3/pgen2/tokenize.py
View File

@ -48,6 +48,10 @@
def group(*choices): return '(' + '|'.join(choices) + ')'
def any(*choices): return group(*choices) + '*'
def maybe(*choices): return group(*choices) + '?'
def _combinations(*l):
return set(
x + y for x in l for y in l + ("",) if x.casefold() != y.casefold()
)
Whitespace = r'[ \f\t]*'
Comment = r'#[^\r\n]*'
@ -74,7 +78,7 @@ def maybe(*choices): return group(*choices) + '?'
Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
# Tail end of """ string.
Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
_litprefix = r"(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?"
_litprefix = r"(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?"
Triple = group(_litprefix + "'''", _litprefix + '"""')
# Single-line ' or " string.
String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
@ -107,59 +111,29 @@ def maybe(*choices): return group(*choices) + '?'
pseudoprog = re.compile(PseudoToken, re.UNICODE)
single3prog = re.compile(Single3)
double3prog = re.compile(Double3)
_strprefixes = (
_combinations('r', 'R', 'f', 'F') |
_combinations('r', 'R', 'b', 'B') |
{'u', 'U', 'ur', 'uR', 'Ur', 'UR'}
)
endprogs = {"'": re.compile(Single), '"': re.compile(Double),
"'''": single3prog, '"""': double3prog,
"r'''": single3prog, 'r"""': double3prog,
"u'''": single3prog, 'u"""': double3prog,
"b'''": single3prog, 'b"""': double3prog,
"f'''": single3prog, 'f"""': double3prog,
"ur'''": single3prog, 'ur"""': double3prog,
"br'''": single3prog, 'br"""': double3prog,
"rb'''": single3prog, 'rb"""': double3prog,
"R'''": single3prog, 'R"""': double3prog,
"U'''": single3prog, 'U"""': double3prog,
"B'''": single3prog, 'B"""': double3prog,
"F'''": single3prog, 'F"""': double3prog,
"uR'''": single3prog, 'uR"""': double3prog,
"Ur'''": single3prog, 'Ur"""': double3prog,
"UR'''": single3prog, 'UR"""': double3prog,
"bR'''": single3prog, 'bR"""': double3prog,
"Br'''": single3prog, 'Br"""': double3prog,
"BR'''": single3prog, 'BR"""': double3prog,
"rB'''": single3prog, 'rB"""': double3prog,
"Rb'''": single3prog, 'Rb"""': double3prog,
"RB'''": single3prog, 'RB"""': double3prog,
'r': None, 'R': None,
'u': None, 'U': None,
'f': None, 'F': None,
'b': None, 'B': None}
**{f"{prefix}'''": single3prog for prefix in _strprefixes},
**{f'{prefix}"""': double3prog for prefix in _strprefixes},
**{prefix: None for prefix in _strprefixes}}
triple_quoted = {}
for t in ("'''", '"""',
"r'''", 'r"""', "R'''", 'R"""',
"u'''", 'u"""', "U'''", 'U"""',
"b'''", 'b"""', "B'''", 'B"""',
"f'''", 'f"""', "F'''", 'F"""',
"ur'''", 'ur"""', "Ur'''", 'Ur"""',
"uR'''", 'uR"""', "UR'''", 'UR"""',
"br'''", 'br"""', "Br'''", 'Br"""',
"bR'''", 'bR"""', "BR'''", 'BR"""',
"rb'''", 'rb"""', "Rb'''", 'Rb"""',
"rB'''", 'rB"""', "RB'''", 'RB"""',):
triple_quoted[t] = t
single_quoted = {}
for t in ("'", '"',
"r'", 'r"', "R'", 'R"',
"u'", 'u"', "U'", 'U"',
"b'", 'b"', "B'", 'B"',
"f'", 'f"', "F'", 'F"',
"ur'", 'ur"', "Ur'", 'Ur"',
"uR'", 'uR"', "UR'", 'UR"',
"br'", 'br"', "Br'", 'Br"',
"bR'", 'bR"', "BR'", 'BR"',
"rb'", 'rb"', "Rb'", 'Rb"',
"rB'", 'rB"', "RB'", 'RB"',):
single_quoted[t] = t
triple_quoted = (
{"'''", '"""'} |
{f"{prefix}'''" for prefix in _strprefixes} |
{f'{prefix}"""' for prefix in _strprefixes}
)
single_quoted = (
{"'", '"'} |
{f"{prefix}'" for prefix in _strprefixes} |
{f'{prefix}"' for prefix in _strprefixes}
)
tabsize = 8

4
tests/python2.py
View File

@ -8,7 +8,7 @@
def function((_globals, _locals)):
exec "print 'hi from exec!'" in _globals, _locals
exec ur"print 'hi from exec!'" in _globals, _locals
function((globals(), locals()))
@ -27,7 +27,7 @@ def function((_globals, _locals)):
def function((_globals, _locals)):
exec "print 'hi from exec!'" in _globals, _locals
exec ur"print 'hi from exec!'" in _globals, _locals
function((globals(), locals()))

2
tests/string_quotes.py
View File

@ -22,6 +22,7 @@
r'Date d\'expiration:(.*)'
r'Tricky "quote'
r'Not-so-tricky \"quote'
rf'{yay}'
'\n\
The \"quick\"\n\
brown fox\n\
@ -56,6 +57,7 @@
r"Date d\'expiration:(.*)"
r'Tricky "quote'
r"Not-so-tricky \"quote"
rf"{yay}"
"\n\
The \"quick\"\n\
brown fox\n\