blib2to3: Never put prefixes on INDENT leaves either

blib2to3/README

@@ -7,5 +7,6 @@ Reasons for forking:
   *args and **kwargs
 - backport of GH-6143 that restores the ability to reformat legacy usage of
   `async`
-- better ability to debug (better reprs for starters)
+- better ability to debug (better reprs)
+- INDENT and DEDENT don't hold whitespace and comment prefixes
 - ability to Cythonize

blib2to3/pgen2/driver.py

@@ -69,7 +69,7 @@ def parse_tokens(self, tokens, debug=False):
             if debug:
                 self.logger.debug("%s %r (prefix=%r)",
                                   token.tok_name[type], value, prefix)
-            if type == token.DEDENT:
+            if type in {token.INDENT, token.DEDENT}:
                 _prefix = prefix
                 prefix = ""
             if p.addtoken(type, value, (prefix, start)):
@@ -77,7 +77,7 @@ def parse_tokens(self, tokens, debug=False):
                     self.logger.debug("Stop.")
                 break
             prefix = ""
-            if type == token.DEDENT:
+            if type in {token.INDENT, token.DEDENT}:
                 prefix = _prefix
             lineno, column = end
             if value.endswith("\n"):

blib2to3/pgen2/tokenize.py

@@ -430,6 +430,10 @@ def generate_tokens(readline):
                 yield stashed
                 stashed = None
 
+            if column > indents[-1]:           # count indents
+                indents.append(column)
+                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
+
             if line[pos] in '#\r\n':           # skip comments or blank lines
                 if line[pos] == '#':
                     comment_token = line[pos:].rstrip('\r\n')
@@ -443,10 +447,7 @@ def generate_tokens(readline):
                            (lnum, pos), (lnum, len(line)), line)
                 continue
 
-            if column > indents[-1]:           # count indents or dedents
+            while column < indents[-1]:         # count dedents
-                indents.append(column)
-                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
-            while column < indents[-1]:
                 if column not in indents:
                     raise IndentationError(
                         "unindent does not match any outer indentation level",

tests/empty_lines.py

@@ -105,6 +105,7 @@ def f():
     if not prev:
         prevp = preceding_leaf(p)
         if not prevp or prevp.type in OPENING_BRACKETS:
+
             return NO
 
         if prevp.type == token.EQUAL: