actions/black
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[blib2to3] Make the grammar pickles faster

Browse Source
This commit is contained in:
Łukasz Langa 2018-04-02 18:28:24 -07:00
parent 65c52a655f
commit 2d80366ac1
1 changed files with 2 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
blib2to3/pgen2/grammar.py
View File

@ -86,21 +86,9 @@ def __init__(self):
self.start = 256 self.start = 256
def dump(self, filename): def dump(self, filename):
"""Dump the grammar tables to a pickle file. """Dump the grammar tables to a pickle file."""
dump() recursively changes all dict to OrderedDict, so the pickled file
is not exactly the same as what was passed in to dump(). load() uses the
pickled file to create the tables, but only changes OrderedDict to dict
at the top level; it does not recursively change OrderedDict to dict.
So, the loaded tables are different from the original tables that were
passed to load() in that some of the OrderedDict (from the pickled file)
are not changed back to dict. For parsing, this has no effect on
performance because OrderedDict uses dict's __getitem__ with nothing in
between.
"""
with open(filename, "wb") as f: with open(filename, "wb") as f:
d = _make_deterministic(self.__dict__) pickle.dump(self.__dict__, f, pickle.HIGHEST_PROTOCOL)
pickle.dump(d, f, 2)
def load(self, filename): def load(self, filename):
"""Load the grammar tables from a pickle file.""" """Load the grammar tables from a pickle file."""
@ -141,17 +129,6 @@ def report(self):
print("start", self.start) print("start", self.start)
def _make_deterministic(top):
if isinstance(top, dict):
return collections.OrderedDict(
sorted(((k, _make_deterministic(v)) for k, v in top.items())))
if isinstance(top, list):
return [_make_deterministic(e) for e in top]
if isinstance(top, tuple):
return tuple(_make_deterministic(e) for e in top)
return top
# Map from operator to number (since tokenize doesn't do this) # Map from operator to number (since tokenize doesn't do this)
opmap_raw = """ opmap_raw = """