Fix parsing of unaligned standalone comments

Fixes #99
Fixes #112

README.md

@@ -491,6 +491,10 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
 
 ## Change Log
 
+### 18.4a2 (unreleased)
+
+* fixed parsing of unaligned standalone comments (#99, #112)
+
 ### 18.4a1
 
 * added `--quiet` (#78)

blib2to3/pgen2/driver.py

@@ -77,6 +77,12 @@ def parse_tokens(self, tokens, debug=False):
                     self.logger.debug("Stop.")
                 break
             prefix = ""
+            if type == token.INDENT:
+                if _prefix.startswith(value):
+                    # Don't double-indent.  Since we're delaying the prefix that
+                    # would normally belong to INDENT, we need to put the value
+                    # at the end versus at the beginning.
+                    _prefix = _prefix[len(value):] + value
             if type in {token.INDENT, token.DEDENT}:
                 prefix = _prefix
             lineno, column = end

blib2to3/pgen2/tokenize.py

@@ -412,10 +412,6 @@ def generate_tokens(readline):
                 yield (NL, line[pos:], (lnum, pos), (lnum, len(line)), line)
                 continue
 
-            if column > indents[-1]:           # count indents
-                indents.append(column)
-                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
-
             if line[pos] == '#':               # skip comments
                 comment_token = line[pos:].rstrip('\r\n')
                 nl_pos = pos + len(comment_token)
@@ -425,6 +421,10 @@ def generate_tokens(readline):
                         (lnum, nl_pos), (lnum, len(line)), line)
                 continue
 
+            if column > indents[-1]:           # count indents
+                indents.append(column)
+                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
+
             while column < indents[-1]:        # count dedents
                 if column not in indents:
                     raise IndentationError(

tests/comments2.py

@@ -23,6 +23,14 @@
     'Generator',
 ]
 
+if 'PYTHON' in os.environ:
+    add_compiler(compiler_from_env())
+else:
+    # for compiler in compilers.values():
+         # add_compiler(compiler)
+    add_compiler(compilers[(7.0, 32)])
+    # add_compiler(compilers[(7.1, 64)])
+
 # Comment before function.
 def inline_comments_in_brackets_ruin_everything():
     if typedargslist:
@@ -97,7 +105,7 @@ def inline_comments_in_brackets_ruin_everything():
             # and round and round we go
         # and round and round we go
 
-    # let's return
+   # let's return
     return Node(
         syms.simple_stmt,
         [
@@ -144,6 +152,14 @@ def inline_comments_in_brackets_ruin_everything():
     "Generator",
 ]
 
+if "PYTHON" in os.environ:
+    add_compiler(compiler_from_env())
+else:
+    # for compiler in compilers.values():
+    # add_compiler(compiler)
+    add_compiler(compilers[(7.0, 32)])
+# add_compiler(compilers[(7.1, 64)])
+
 # Comment before function.