actions/black
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix parsing of unaligned standalone comments

Browse Source 
Fixes #99
Fixes #112
This commit is contained in:
Łukasz Langa 2018-04-11 23:07:56 -07:00
parent 1909d8cc6b
commit 9138a75b75
4 changed files with 31 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -491,6 +491,10 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
## Change Log
### 18.4a2 (unreleased)
* fixed parsing of unaligned standalone comments (#99, #112)
### 18.4a1
* added `--quiet` (#78)

6
blib2to3/pgen2/driver.py
View File

@ -77,6 +77,12 @@ def parse_tokens(self, tokens, debug=False):
self.logger.debug("Stop.")
break
prefix = ""
if type == token.INDENT:
if _prefix.startswith(value):
# Don't double-indent. Since we're delaying the prefix that
# would normally belong to INDENT, we need to put the value
# at the end versus at the beginning.
_prefix = _prefix[len(value):] + value
if type in {token.INDENT, token.DEDENT}:
prefix = _prefix
lineno, column = end

8
blib2to3/pgen2/tokenize.py
View File

@ -412,10 +412,6 @@ def generate_tokens(readline):
yield (NL, line[pos:], (lnum, pos), (lnum, len(line)), line)
continue
if column > indents[-1]: # count indents
indents.append(column)
yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
if line[pos] == '#': # skip comments
comment_token = line[pos:].rstrip('\r\n')
nl_pos = pos + len(comment_token)
@ -425,6 +421,10 @@ def generate_tokens(readline):
(lnum, nl_pos), (lnum, len(line)), line)
continue
if column > indents[-1]: # count indents
indents.append(column)
yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
while column < indents[-1]: # count dedents
if column not in indents:
raise IndentationError(

18
tests/comments2.py
View File

@ -23,6 +23,14 @@
'Generator',
]
if 'PYTHON' in os.environ:
add_compiler(compiler_from_env())
else:
# for compiler in compilers.values():
# add_compiler(compiler)
add_compiler(compilers[(7.0, 32)])
# add_compiler(compilers[(7.1, 64)])
# Comment before function.
def inline_comments_in_brackets_ruin_everything():
if typedargslist:
@ -97,7 +105,7 @@ def inline_comments_in_brackets_ruin_everything():
# and round and round we go
# and round and round we go
# let's return
# let's return
return Node(
syms.simple_stmt,
[
@ -144,6 +152,14 @@ def inline_comments_in_brackets_ruin_everything():
"Generator",
]
if "PYTHON" in os.environ:
add_compiler(compiler_from_env())
else:
# for compiler in compilers.values():
# add_compiler(compiler)
add_compiler(compilers[(7.0, 32)])
# add_compiler(compilers[(7.1, 64)])
# Comment before function.