Initial commit
This commit is contained in:
Łukasz Langa
commit
e74117f172
8
.flake8
Normal file
8
.flake8
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
# This is an example .flake8 config, used when developing *Black* itself.
|
||||||
|
# Keep in sync with setup.cfg which is used for source packages.
|
||||||
|
|
||||||
|
[flake8]
|
||||||
|
ignore = E266, E501
|
||||||
|
max-line-length = 80
|
||||||
|
max-complexity = 12
|
||||||
|
select = B,C,E,F,W,T4,B9
|
||||||
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
|||||||
|
.coverage
|
||||||
15
.travis.yml
Normal file
15
.travis.yml
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
sudo: false
|
||||||
|
language: python
|
||||||
|
before_script:
|
||||||
|
- pip install -e .
|
||||||
|
# test script
|
||||||
|
script: python setup.py test
|
||||||
|
notifications:
|
||||||
|
on_success: change
|
||||||
|
on_failure: always
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
- python: 3.6
|
||||||
|
- python: 3.6-dev
|
||||||
|
- python: 3.7-dev
|
||||||
|
- python: 3.8-dev
|
||||||
21
LICENSE
Normal file
21
LICENSE
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
The MIT License (MIT)
|
||||||
|
|
||||||
|
Copyright (c) 2018 Łukasz Langa
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
2
MANIFEST.in
Normal file
2
MANIFEST.in
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
include *.rst *.md LICENSE
|
||||||
|
recursive-include tests *.txt *.py
|
||||||
17
Pipfile
Normal file
17
Pipfile
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
[[source]]
|
||||||
|
url = "https://pypi.python.org/simple"
|
||||||
|
verify_ssl = true
|
||||||
|
name = "pypi"
|
||||||
|
|
||||||
|
[packages]
|
||||||
|
attrs = "*"
|
||||||
|
click = "*"
|
||||||
|
|
||||||
|
[dev-packages]
|
||||||
|
coverage = "*"
|
||||||
|
flake8 = "*"
|
||||||
|
flake8-bugbear = "*"
|
||||||
|
flake8-mypy = "*"
|
||||||
|
mypy = "*"
|
||||||
|
pypandoc = "*"
|
||||||
|
twine = "*"
|
||||||
243
Pipfile.lock
generated
Normal file
243
Pipfile.lock
generated
Normal file
@ -0,0 +1,243 @@
|
|||||||
|
{
|
||||||
|
"_meta": {
|
||||||
|
"hash": {
|
||||||
|
"sha256": "e2dc877c2f32df83197fc3dc0f49e0a66d0d099aab106b99d64fdbe5b14cc91b"
|
||||||
|
},
|
||||||
|
"host-environment-markers": {
|
||||||
|
"implementation_name": "cpython",
|
||||||
|
"implementation_version": "3.6.4",
|
||||||
|
"os_name": "posix",
|
||||||
|
"platform_machine": "x86_64",
|
||||||
|
"platform_python_implementation": "CPython",
|
||||||
|
"platform_release": "17.4.0",
|
||||||
|
"platform_system": "Darwin",
|
||||||
|
"platform_version": "Darwin Kernel Version 17.4.0: Sun Dec 17 09:19:54 PST 2017; root:xnu-4570.41.2~1/RELEASE_X86_64",
|
||||||
|
"python_full_version": "3.6.4",
|
||||||
|
"python_version": "3.6",
|
||||||
|
"sys_platform": "darwin"
|
||||||
|
},
|
||||||
|
"pipfile-spec": 6,
|
||||||
|
"requires": {},
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"name": "pypi",
|
||||||
|
"url": "https://pypi.python.org/simple",
|
||||||
|
"verify_ssl": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"default": {
|
||||||
|
"attrs": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:a17a9573a6f475c99b551c0e0a812707ddda1ec9653bed04c13841404ed6f450",
|
||||||
|
"sha256:1c7960ccfd6a005cd9f7ba884e6316b5e430a3f1a6c37c5f87d8b43f83b54ec9"
|
||||||
|
],
|
||||||
|
"version": "==17.4.0"
|
||||||
|
},
|
||||||
|
"click": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d",
|
||||||
|
"sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b"
|
||||||
|
],
|
||||||
|
"version": "==6.7"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"develop": {
|
||||||
|
"attrs": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:a17a9573a6f475c99b551c0e0a812707ddda1ec9653bed04c13841404ed6f450",
|
||||||
|
"sha256:1c7960ccfd6a005cd9f7ba884e6316b5e430a3f1a6c37c5f87d8b43f83b54ec9"
|
||||||
|
],
|
||||||
|
"version": "==17.4.0"
|
||||||
|
},
|
||||||
|
"certifi": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:14131608ad2fd56836d33a71ee60fa1c82bc9d2c8d98b7bdbc631fe1b3cd1296",
|
||||||
|
"sha256:edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d"
|
||||||
|
],
|
||||||
|
"version": "==2018.1.18"
|
||||||
|
},
|
||||||
|
"chardet": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691",
|
||||||
|
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"
|
||||||
|
],
|
||||||
|
"version": "==3.0.4"
|
||||||
|
},
|
||||||
|
"coverage": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:7608a3dd5d73cb06c531b8925e0ef8d3de31fed2544a7de6c63960a1e73ea4bc",
|
||||||
|
"sha256:3a2184c6d797a125dca8367878d3b9a178b6fdd05fdc2d35d758c3006a1cd694",
|
||||||
|
"sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80",
|
||||||
|
"sha256:0b136648de27201056c1869a6c0d4e23f464750fd9a9ba9750b8336a244429ed",
|
||||||
|
"sha256:337ded681dd2ef9ca04ef5d93cfc87e52e09db2594c296b4a0a3662cb1b41249",
|
||||||
|
"sha256:3eb42bf89a6be7deb64116dd1cc4b08171734d721e7a7e57ad64cc4ef29ed2f1",
|
||||||
|
"sha256:be6cfcd8053d13f5f5eeb284aa8a814220c3da1b0078fa859011c7fffd86dab9",
|
||||||
|
"sha256:69bf008a06b76619d3c3f3b1983f5145c75a305a0fea513aca094cae5c40a8f5",
|
||||||
|
"sha256:2eb564bbf7816a9d68dd3369a510be3327f1c618d2357fa6b1216994c2e3d508",
|
||||||
|
"sha256:9d6dd10d49e01571bf6e147d3b505141ffc093a06756c60b053a859cb2128b1f",
|
||||||
|
"sha256:701cd6093d63e6b8ad7009d8a92425428bc4d6e7ab8d75efbb665c806c1d79ba",
|
||||||
|
"sha256:5a13ea7911ff5e1796b6d5e4fbbf6952381a611209b736d48e675c2756f3f74e",
|
||||||
|
"sha256:c1bb572fab8208c400adaf06a8133ac0712179a334c09224fb11393e920abcdd",
|
||||||
|
"sha256:03481e81d558d30d230bc12999e3edffe392d244349a90f4ef9b88425fac74ba",
|
||||||
|
"sha256:28b2191e7283f4f3568962e373b47ef7f0392993bb6660d079c62bd50fe9d162",
|
||||||
|
"sha256:de4418dadaa1c01d497e539210cb6baa015965526ff5afc078c57ca69160108d",
|
||||||
|
"sha256:8c3cb8c35ec4d9506979b4cf90ee9918bc2e49f84189d9bf5c36c0c1119c6558",
|
||||||
|
"sha256:7e1fe19bd6dce69d9fd159d8e4a80a8f52101380d5d3a4d374b6d3eae0e5de9c",
|
||||||
|
"sha256:6bc583dc18d5979dc0f6cec26a8603129de0304d5ae1f17e57a12834e7235062",
|
||||||
|
"sha256:198626739a79b09fa0a2f06e083ffd12eb55449b5f8bfdbeed1df4910b2ca640",
|
||||||
|
"sha256:7aa36d2b844a3e4a4b356708d79fd2c260281a7390d678a10b91ca595ddc9e99",
|
||||||
|
"sha256:3d72c20bd105022d29b14a7d628462ebdc61de2f303322c0212a054352f3b287",
|
||||||
|
"sha256:4635a184d0bbe537aa185a34193898eee409332a8ccb27eea36f262566585000",
|
||||||
|
"sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6",
|
||||||
|
"sha256:76ecd006d1d8f739430ec50cc872889af1f9c1b6b8f48e29941814b09b0fd3cc",
|
||||||
|
"sha256:7d3f553904b0c5c016d1dad058a7554c7ac4c91a789fca496e7d8347ad040653",
|
||||||
|
"sha256:3c79a6f7b95751cdebcd9037e4d06f8d5a9b60e4ed0cd231342aa8ad7124882a",
|
||||||
|
"sha256:56e448f051a201c5ebbaa86a5efd0ca90d327204d8b059ab25ad0f35fbfd79f1",
|
||||||
|
"sha256:ac4fef68da01116a5c117eba4dd46f2e06847a497de5ed1d64bb99a5fda1ef91",
|
||||||
|
"sha256:1c383d2ef13ade2acc636556fd544dba6e14fa30755f26812f54300e401f98f2",
|
||||||
|
"sha256:b8815995e050764c8610dbc82641807d196927c3dbed207f0a079833ffcf588d",
|
||||||
|
"sha256:104ab3934abaf5be871a583541e8829d6c19ce7bde2923b2751e0d3ca44db60a",
|
||||||
|
"sha256:9e112fcbe0148a6fa4f0a02e8d58e94470fc6cb82a5481618fea901699bf34c4",
|
||||||
|
"sha256:15b111b6a0f46ee1a485414a52a7ad1d703bdf984e9ed3c288a4414d3871dcbd",
|
||||||
|
"sha256:e4d96c07229f58cb686120f168276e434660e4358cc9cf3b0464210b04913e77",
|
||||||
|
"sha256:f8a923a85cb099422ad5a2e345fe877bbc89a8a8b23235824a93488150e45f6e"
|
||||||
|
],
|
||||||
|
"version": "==4.5.1"
|
||||||
|
},
|
||||||
|
"flake8": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:c7841163e2b576d435799169b78703ad6ac1bbb0f199994fc05f700b2a90ea37",
|
||||||
|
"sha256:7253265f7abd8b313e3892944044a365e3f4ac3fcdcfb4298f55ee9ddf188ba0"
|
||||||
|
],
|
||||||
|
"version": "==3.5.0"
|
||||||
|
},
|
||||||
|
"flake8-bugbear": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:541746f0f3b2f1a8d7278e1d2d218df298996b60b02677708560db7c7e620e3b",
|
||||||
|
"sha256:5f14a99d458e29cb92be9079c970030e0dd398b2decb179d76d39a5266ea1578"
|
||||||
|
],
|
||||||
|
"version": "==18.2.0"
|
||||||
|
},
|
||||||
|
"flake8-mypy": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:cff009f4250e8391bf48990093cff85802778c345c8449d6498b62efefeebcbc",
|
||||||
|
"sha256:47120db63aff631ee1f84bac6fe8e64731dc66da3efc1c51f85e15ade4a3ba18"
|
||||||
|
],
|
||||||
|
"version": "==17.8.0"
|
||||||
|
},
|
||||||
|
"idna": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4",
|
||||||
|
"sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f"
|
||||||
|
],
|
||||||
|
"version": "==2.6"
|
||||||
|
},
|
||||||
|
"mccabe": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
|
||||||
|
"sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
|
||||||
|
],
|
||||||
|
"version": "==0.6.1"
|
||||||
|
},
|
||||||
|
"mypy": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:884f18f3a40cfcf24cdd5860b84958cfb35e6563e439c5adc1503878df221dc3",
|
||||||
|
"sha256:83d798f66323f2de6191d66d9ae5ab234e4ee5b400010e19c58d75d308049f25"
|
||||||
|
],
|
||||||
|
"version": "==0.570"
|
||||||
|
},
|
||||||
|
"pkginfo": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:31a49103180ae1518b65d3f4ce09c784e2bc54e338197668b4fb7dc539521024",
|
||||||
|
"sha256:bb1a6aeabfc898f5df124e7e00303a5b3ec9a489535f346bfbddb081af93f89e"
|
||||||
|
],
|
||||||
|
"version": "==1.4.1"
|
||||||
|
},
|
||||||
|
"pycodestyle": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:6c4245ade1edfad79c3446fadfc96b0de2759662dc29d07d80a6f27ad1ca6ba9",
|
||||||
|
"sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766"
|
||||||
|
],
|
||||||
|
"version": "==2.3.1"
|
||||||
|
},
|
||||||
|
"pyflakes": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:08bd6a50edf8cffa9fa09a463063c425ecaaf10d1eb0335a7e8b1401aef89e6f",
|
||||||
|
"sha256:8d616a382f243dbf19b54743f280b80198be0bca3a5396f1d2e1fca6223e8805"
|
||||||
|
],
|
||||||
|
"version": "==1.6.0"
|
||||||
|
},
|
||||||
|
"pypandoc": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:e914e6d5f84a76764887e4d909b09d63308725f0cbb5293872c2c92f07c11a5b"
|
||||||
|
],
|
||||||
|
"version": "==1.4"
|
||||||
|
},
|
||||||
|
"requests": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b",
|
||||||
|
"sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e"
|
||||||
|
],
|
||||||
|
"version": "==2.18.4"
|
||||||
|
},
|
||||||
|
"requests-toolbelt": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237",
|
||||||
|
"sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5"
|
||||||
|
],
|
||||||
|
"version": "==0.8.0"
|
||||||
|
},
|
||||||
|
"tqdm": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:f66468c14ccd011a627734c9b3fd72f20ce16f8faecc47384eb2507af5924fb9",
|
||||||
|
"sha256:5ec0d4442358e55cdb4a0471d04c6c831518fd8837f259db5537d90feab380df"
|
||||||
|
],
|
||||||
|
"version": "==4.19.6"
|
||||||
|
},
|
||||||
|
"twine": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:d3ce5c480c22ccfb761cd358526e862b32546d2fe4bc93d46b5cf04ea3cc46ca",
|
||||||
|
"sha256:caa45b7987fc96321258cd7668e3be2ff34064f5c66d2d975b641adca659c1ab"
|
||||||
|
],
|
||||||
|
"version": "==1.9.1"
|
||||||
|
},
|
||||||
|
"typed-ast": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:0948004fa228ae071054f5208840a1e88747a357ec1101c17217bfe99b299d58",
|
||||||
|
"sha256:25d8feefe27eb0303b73545416b13d108c6067b846b543738a25ff304824ed9a",
|
||||||
|
"sha256:c05b41bc1deade9f90ddc5d988fe506208019ebba9f2578c622516fd201f5863",
|
||||||
|
"sha256:519425deca5c2b2bdac49f77b2c5625781abbaf9a809d727d3a5596b30bb4ded",
|
||||||
|
"sha256:6de012d2b166fe7a4cdf505eee3aaa12192f7ba365beeefaca4ec10e31241a85",
|
||||||
|
"sha256:79b91ebe5a28d349b6d0d323023350133e927b4de5b651a8aa2db69c761420c6",
|
||||||
|
"sha256:a8034021801bc0440f2e027c354b4eafd95891b573e12ff0418dec385c76785c",
|
||||||
|
"sha256:f19f2a4f547505fe9072e15f6f4ae714af51b5a681a97f187971f50c283193b6",
|
||||||
|
"sha256:c9b060bd1e5a26ab6e8267fd46fc9e02b54eb15fffb16d112d4c7b1c12987559",
|
||||||
|
"sha256:2e214b72168ea0275efd6c884b114ab42e316de3ffa125b267e732ed2abda892",
|
||||||
|
"sha256:bc978ac17468fe868ee589c795d06777f75496b1ed576d308002c8a5756fb9ea",
|
||||||
|
"sha256:edb04bdd45bfd76c8292c4d9654568efaedf76fe78eb246dde69bdb13b2dad87",
|
||||||
|
"sha256:668d0cec391d9aed1c6a388b0d5b97cd22e6073eaa5fbaa6d2946603b4871efe",
|
||||||
|
"sha256:29464a177d56e4e055b5f7b629935af7f49c196be47528cc94e0a7bf83fbc2b9",
|
||||||
|
"sha256:8550177fa5d4c1f09b5e5f524411c44633c80ec69b24e0e98906dd761941ca46",
|
||||||
|
"sha256:3e0d5e48e3a23e9a4d1a9f698e32a542a4a288c871d33ed8df1b092a40f3a0f9",
|
||||||
|
"sha256:68ba70684990f59497680ff90d18e756a47bf4863c604098f10de9716b2c0bdd",
|
||||||
|
"sha256:57fe287f0cdd9ceaf69e7b71a2e94a24b5d268b35df251a88fef5cc241bf73aa"
|
||||||
|
],
|
||||||
|
"version": "==1.1.0"
|
||||||
|
},
|
||||||
|
"urllib3": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b",
|
||||||
|
"sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f"
|
||||||
|
],
|
||||||
|
"version": "==1.22"
|
||||||
|
},
|
||||||
|
"wheel": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:e721e53864f084f956f40f96124a74da0631ac13fbbd1ba99e8e2b5e9cafdf64",
|
||||||
|
"sha256:9515fe0a94e823fd90b08d22de45d7bde57c90edce705b22f5e1ecf7e1b653c8"
|
||||||
|
],
|
||||||
|
"version": "==0.30.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
260
README.md
Normal file
260
README.md
Normal file
@ -0,0 +1,260 @@
|
|||||||
|
# black
|
||||||
|
|
||||||
|
[](https://travis-ci.org/ambv/black)
|
||||||
|
|
||||||
|
> Any color you like.
|
||||||
|
|
||||||
|
|
||||||
|
*Black* is the uncompromising Python code formatter. By using it, you
|
||||||
|
agree to cease control over minutiae of hand-formatting. In return,
|
||||||
|
*Black* gives you speed, determinism, and freedom from `pycodestyle`
|
||||||
|
nagging about formatting. You will save time and mental energy for
|
||||||
|
more important matters.
|
||||||
|
|
||||||
|
Blackened code looks the same regardless of the project you're reading.
|
||||||
|
Formatting becomes transparent after a while and you can focus on the
|
||||||
|
content instead.
|
||||||
|
|
||||||
|
*Black* makes code review faster by producing the smallest diffs
|
||||||
|
possible.
|
||||||
|
|
||||||
|
|
||||||
|
## NOTE: This is an early pre-release
|
||||||
|
|
||||||
|
*Black* can already successfully format itself and the standard library.
|
||||||
|
It also sports a decent test suite. However, it is still very new.
|
||||||
|
Things will probably be wonky for a while. This is made explicit by the
|
||||||
|
"Alpha" trove classifier, as well as by the "a" in the version number.
|
||||||
|
What this means for you is that **until the formatter becomes stable,
|
||||||
|
you should expect some formatting to change in the future**.
|
||||||
|
|
||||||
|
Also, as a temporary safety measure, *Black* will check that the
|
||||||
|
reformatted code still produces a valid AST that is equivalent to the
|
||||||
|
original. This slows it down. If you're feeling confident, use
|
||||||
|
``--fast``.
|
||||||
|
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
*Black* can be installed by running `pip install black`.
|
||||||
|
|
||||||
|
```
|
||||||
|
black [OPTIONS] [SRC]...
|
||||||
|
|
||||||
|
Options:
|
||||||
|
-l, --line-length INTEGER Where to wrap around. [default: 88]
|
||||||
|
--fast / --safe If --fast given, skip temporary sanity checks.
|
||||||
|
[default: --safe]
|
||||||
|
--version Show the version and exit.
|
||||||
|
--help Show this message and exit.
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
## The philosophy behind *Black*
|
||||||
|
|
||||||
|
*Black* reformats entire files in place. It is not configurable. It
|
||||||
|
doesn't take previous formatting into account. It doesn't reformat
|
||||||
|
blocks that start with `# fmt: off` and end with `# fmt: on`. It also
|
||||||
|
recognizes [YAPF](https://github.com/google/yapf)'s block comments to
|
||||||
|
the same effect, as a courtesy for straddling code.
|
||||||
|
|
||||||
|
|
||||||
|
### How *Black* formats files
|
||||||
|
|
||||||
|
*Black* ignores previous formatting and applies uniform horizontal
|
||||||
|
and vertical whitespace to your code. The rules for horizontal
|
||||||
|
whitespace are pretty obvious and can be summarized as: do whatever
|
||||||
|
makes `pycodestyle` happy.
|
||||||
|
|
||||||
|
As for vertical whitespace, *Black* tries to render one full expression
|
||||||
|
or simple statement per line. If this fits the allotted line length,
|
||||||
|
great.
|
||||||
|
```!py3
|
||||||
|
# in:
|
||||||
|
l = [1,
|
||||||
|
2,
|
||||||
|
3,
|
||||||
|
]
|
||||||
|
|
||||||
|
# out:
|
||||||
|
l = [1, 2, 3]
|
||||||
|
```
|
||||||
|
|
||||||
|
If not, *Black* will look at the contents of the first outer matching
|
||||||
|
brackets and put that in a separate indented line.
|
||||||
|
```!py3
|
||||||
|
# in:
|
||||||
|
l = [[n for n in list_bosses()], [n for n in list_employees()]]
|
||||||
|
|
||||||
|
# out:
|
||||||
|
l = [
|
||||||
|
[n for n in list_bosses()], [n for n in list_employees()]
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
If that still doesn't fit the bill, it will decompose the internal
|
||||||
|
expression further using the same rule, indenting matching brackets
|
||||||
|
every time. If the contents of the matching brackets pair are
|
||||||
|
comma-separated (like an argument list, or a dict literal, and so on)
|
||||||
|
then *Black* will first try to keep them on the same line with the
|
||||||
|
matching brackets. If that doesn't work, it will put all of them in
|
||||||
|
separate lines.
|
||||||
|
```!py3
|
||||||
|
# in:
|
||||||
|
def very_important_function(template: str, *variables, *, file: os.PathLike, debug: bool = False):
|
||||||
|
"""Applies `variables` to the `template` and writes to `file`."""
|
||||||
|
with open(file, 'w') as f:
|
||||||
|
...
|
||||||
|
|
||||||
|
# out:
|
||||||
|
def very_important_function(
|
||||||
|
template: str,
|
||||||
|
*variables,
|
||||||
|
*,
|
||||||
|
file: os.PathLike,
|
||||||
|
debug: bool = False,
|
||||||
|
):
|
||||||
|
"""Applies `variables` to the `template` and writes to `file`."""
|
||||||
|
with open(file, 'w') as f:
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
You might have noticed that closing brackets are always dedented and
|
||||||
|
that a trailing comma is always added. Such formatting produces smaller
|
||||||
|
diffs; when you add or remove an element, it's always just one line.
|
||||||
|
Also, having the closing bracket dedented provides a clear delimiter
|
||||||
|
between two distinct sections of the code that otherwise share the same
|
||||||
|
indentation level (like the arguments list and the docstring in the
|
||||||
|
example above).
|
||||||
|
|
||||||
|
Unnecessary trailing commas are removed if an expression fits in one
|
||||||
|
line. This makes it 1% more likely that your line won't exceed the
|
||||||
|
allotted line length limit.
|
||||||
|
|
||||||
|
*Black* avoids spurious vertical whitespace. This is in the spirit of
|
||||||
|
PEP 8 which says that in-function vertical whitespace should only be
|
||||||
|
used sparingly. One exception is control flow statements: *Black* will
|
||||||
|
always emit an extra empty line after ``return``, ``raise``, ``break``,
|
||||||
|
``continue``, and ``yield``. This is to make changes in control flow
|
||||||
|
more prominent to readers of your code.
|
||||||
|
|
||||||
|
That's it. The rest of the whitespace formatting rules follow PEP 8 and
|
||||||
|
are designed to keep `pycodestyle` quiet.
|
||||||
|
|
||||||
|
|
||||||
|
### Line length
|
||||||
|
|
||||||
|
You probably noticed the peculiar default line length. *Black* defaults
|
||||||
|
to 88 characters per line, which happens to be 10% over 80. This number
|
||||||
|
was found to produce significantly shorter files than sticking with 80
|
||||||
|
(the most popular), or even 79 (used by the standard library). In
|
||||||
|
general, [90-ish seems like the wise choice](https://youtu.be/wf-BqAjZb8M?t=260).
|
||||||
|
|
||||||
|
If you're paid by the line of code you write, you can pass
|
||||||
|
`--line-length` with a lower number. *Black* will try to respect that.
|
||||||
|
However, sometimes it won't be able to without breaking other rules. In
|
||||||
|
those rare cases, auto-formatted code will exceed your allotted limit.
|
||||||
|
|
||||||
|
You can also increase it, but remember that people with sight disabilities
|
||||||
|
find it harder to work with line lengths exceeding 100 characters.
|
||||||
|
It also adversely affects side-by-side diff review on typical screen
|
||||||
|
resolutions. Long lines also make it harder to present code neatly
|
||||||
|
in documentation or talk slides.
|
||||||
|
|
||||||
|
If you're using Flake8, you can bump `max-line-length` to 88 and forget
|
||||||
|
about it. Alternatively, use [Bugbear](https://github.com/PyCQA/flake8-bugbear)'s
|
||||||
|
B950 warning instead of E501 and keep the max line length at 80 which
|
||||||
|
you are probably already using. You'd do it like this:
|
||||||
|
```!ini
|
||||||
|
[flake8]
|
||||||
|
max-line-length = 80
|
||||||
|
...
|
||||||
|
select = C,E,F,W,B,B950
|
||||||
|
ignore = E501
|
||||||
|
```
|
||||||
|
|
||||||
|
You'll find *Black*'s own .flake8 config file is configured like this.
|
||||||
|
If you're curious about the reasoning behind B950, Bugbear's documentation
|
||||||
|
explains it. The tl;dr is "it's like highway speed limits, we won't
|
||||||
|
bother you if you overdo it by a few km/h".
|
||||||
|
|
||||||
|
|
||||||
|
### Editor integration
|
||||||
|
|
||||||
|
There is currently no integration with any text editors. Vim and
|
||||||
|
Atom/Nuclide integration is planned by the author, others will require
|
||||||
|
external contributions.
|
||||||
|
|
||||||
|
Patches welcome! ✨ 🍰 ✨
|
||||||
|
|
||||||
|
|
||||||
|
## Testimonials
|
||||||
|
|
||||||
|
**Dusty Phillips**, [writer](https://smile.amazon.com/s/ref=nb_sb_noss?url=search-alias%3Daps&field-keywords=dusty+phillips):
|
||||||
|
|
||||||
|
> Black is opinionated so you don't have to be.
|
||||||
|
|
||||||
|
**Hynek Schlawack**, [creator of `attrs`](http://www.attrs.org/), core
|
||||||
|
developer of Twisted and CPython:
|
||||||
|
|
||||||
|
> An auto-formatter that doesn't suck is all I want for Xmas!
|
||||||
|
|
||||||
|
**Carl Meyer**, [Django](https://www.djangoproject.com/) core developer:
|
||||||
|
|
||||||
|
> At least the name is good.
|
||||||
|
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
Just run:
|
||||||
|
|
||||||
|
```
|
||||||
|
python setup.py test
|
||||||
|
```
|
||||||
|
|
||||||
|
## This tool requires Python 3.6.0+ to run
|
||||||
|
|
||||||
|
But you can reformat Python 2 code with it, too. *Black* is able to parse
|
||||||
|
all of the new syntax supported on Python 3.6 but also *effectively all*
|
||||||
|
the Python 2 syntax at the same time, as long as you're not using print
|
||||||
|
statements.
|
||||||
|
|
||||||
|
By making the code exclusively Python 3.6+, I'm able to focus on the
|
||||||
|
quality of the formatting and re-use all the nice features of the new
|
||||||
|
releases (check out [pathlib](docs.python.org/3/library/pathlib.html) or
|
||||||
|
f-strings) instead of wasting cycles on Unicode compatibility, and so on.
|
||||||
|
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
MIT
|
||||||
|
|
||||||
|
|
||||||
|
## Contributing
|
||||||
|
|
||||||
|
In terms of inspiration, *Black* is about as configurable as *gofmt* and
|
||||||
|
*rustfmt* are. This is deliberate.
|
||||||
|
|
||||||
|
Bug reports and fixes are always welcome! However, before you suggest a
|
||||||
|
new feature or configuration knob, ask yourself why you want it. If it
|
||||||
|
enables better integration with some workflow, fixes an inconsistency,
|
||||||
|
speeds things up, and so on - go for it! On the other hand, if your
|
||||||
|
answer is "because I don't like a particular formatting" then you're not
|
||||||
|
ready to embrace *Black* yet. Such changes are unlikely to get accepted.
|
||||||
|
You can still try but prepare to be disappointed.
|
||||||
|
|
||||||
|
|
||||||
|
## Change Log
|
||||||
|
|
||||||
|
### 18.3a0
|
||||||
|
|
||||||
|
* first published version, Happy 🍰 Day 2018!
|
||||||
|
|
||||||
|
* alpha quality
|
||||||
|
|
||||||
|
* date-versioned (see: http://calver.org/)
|
||||||
|
|
||||||
|
|
||||||
|
## Authors
|
||||||
|
|
||||||
|
Glued together by [Łukasz Langa](mailto:lukasz@langa.pl).
|
||||||
173
blib2to3/Grammar.txt
Normal file
173
blib2to3/Grammar.txt
Normal file
@ -0,0 +1,173 @@
|
|||||||
|
# Grammar for 2to3. This grammar supports Python 2.x and 3.x.
|
||||||
|
|
||||||
|
# NOTE WELL: You should also follow all the steps listed at
|
||||||
|
# https://devguide.python.org/grammar/
|
||||||
|
|
||||||
|
# Start symbols for the grammar:
|
||||||
|
# file_input is a module or sequence of commands read from an input file;
|
||||||
|
# single_input is a single interactive statement;
|
||||||
|
# eval_input is the input for the eval() and input() functions.
|
||||||
|
# NB: compound_stmt in single_input is followed by extra NEWLINE!
|
||||||
|
file_input: (NEWLINE | stmt)* ENDMARKER
|
||||||
|
single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
|
||||||
|
eval_input: testlist NEWLINE* ENDMARKER
|
||||||
|
|
||||||
|
decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
|
||||||
|
decorators: decorator+
|
||||||
|
decorated: decorators (classdef | funcdef | async_funcdef)
|
||||||
|
async_funcdef: 'async' funcdef
|
||||||
|
funcdef: 'def' NAME parameters ['->' test] ':' suite
|
||||||
|
parameters: '(' [typedargslist] ')'
|
||||||
|
typedargslist: ((tfpdef ['=' test] ',')*
|
||||||
|
('*' [tname] (',' tname ['=' test])* [',' ['**' tname [',']]] | '**' tname [','])
|
||||||
|
| tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
|
||||||
|
tname: NAME [':' test]
|
||||||
|
tfpdef: tname | '(' tfplist ')'
|
||||||
|
tfplist: tfpdef (',' tfpdef)* [',']
|
||||||
|
varargslist: ((vfpdef ['=' test] ',')*
|
||||||
|
('*' [vname] (',' vname ['=' test])* [',' ['**' vname [',']]] | '**' vname [','])
|
||||||
|
| vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
|
||||||
|
vname: NAME
|
||||||
|
vfpdef: vname | '(' vfplist ')'
|
||||||
|
vfplist: vfpdef (',' vfpdef)* [',']
|
||||||
|
|
||||||
|
stmt: simple_stmt | compound_stmt
|
||||||
|
simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
|
||||||
|
small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt |
|
||||||
|
import_stmt | global_stmt | exec_stmt | assert_stmt)
|
||||||
|
expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
|
||||||
|
('=' (yield_expr|testlist_star_expr))*)
|
||||||
|
annassign: ':' test ['=' test]
|
||||||
|
testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
|
||||||
|
augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
|
||||||
|
'<<=' | '>>=' | '**=' | '//=')
|
||||||
|
# For normal and annotated assignments, additional restrictions enforced by the interpreter
|
||||||
|
print_stmt: 'print' ( [ test (',' test)* [','] ] |
|
||||||
|
'>>' test [ (',' test)+ [','] ] )
|
||||||
|
del_stmt: 'del' exprlist
|
||||||
|
pass_stmt: 'pass'
|
||||||
|
flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
|
||||||
|
break_stmt: 'break'
|
||||||
|
continue_stmt: 'continue'
|
||||||
|
return_stmt: 'return' [testlist]
|
||||||
|
yield_stmt: yield_expr
|
||||||
|
raise_stmt: 'raise' [test ['from' test | ',' test [',' test]]]
|
||||||
|
import_stmt: import_name | import_from
|
||||||
|
import_name: 'import' dotted_as_names
|
||||||
|
import_from: ('from' ('.'* dotted_name | '.'+)
|
||||||
|
'import' ('*' | '(' import_as_names ')' | import_as_names))
|
||||||
|
import_as_name: NAME ['as' NAME]
|
||||||
|
dotted_as_name: dotted_name ['as' NAME]
|
||||||
|
import_as_names: import_as_name (',' import_as_name)* [',']
|
||||||
|
dotted_as_names: dotted_as_name (',' dotted_as_name)*
|
||||||
|
dotted_name: NAME ('.' NAME)*
|
||||||
|
global_stmt: ('global' | 'nonlocal') NAME (',' NAME)*
|
||||||
|
exec_stmt: 'exec' expr ['in' test [',' test]]
|
||||||
|
assert_stmt: 'assert' test [',' test]
|
||||||
|
|
||||||
|
compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
|
||||||
|
async_stmt: 'async' (funcdef | with_stmt | for_stmt)
|
||||||
|
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
|
||||||
|
while_stmt: 'while' test ':' suite ['else' ':' suite]
|
||||||
|
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
|
||||||
|
try_stmt: ('try' ':' suite
|
||||||
|
((except_clause ':' suite)+
|
||||||
|
['else' ':' suite]
|
||||||
|
['finally' ':' suite] |
|
||||||
|
'finally' ':' suite))
|
||||||
|
with_stmt: 'with' with_item (',' with_item)* ':' suite
|
||||||
|
with_item: test ['as' expr]
|
||||||
|
with_var: 'as' expr
|
||||||
|
# NB compile.c makes sure that the default except clause is last
|
||||||
|
except_clause: 'except' [test [(',' | 'as') test]]
|
||||||
|
suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
|
||||||
|
|
||||||
|
# Backward compatibility cruft to support:
|
||||||
|
# [ x for x in lambda: True, lambda: False if x() ]
|
||||||
|
# even while also allowing:
|
||||||
|
# lambda x: 5 if x else 2
|
||||||
|
# (But not a mix of the two)
|
||||||
|
testlist_safe: old_test [(',' old_test)+ [',']]
|
||||||
|
old_test: or_test | old_lambdef
|
||||||
|
old_lambdef: 'lambda' [varargslist] ':' old_test
|
||||||
|
|
||||||
|
test: or_test ['if' or_test 'else' test] | lambdef
|
||||||
|
or_test: and_test ('or' and_test)*
|
||||||
|
and_test: not_test ('and' not_test)*
|
||||||
|
not_test: 'not' not_test | comparison
|
||||||
|
comparison: expr (comp_op expr)*
|
||||||
|
comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
|
||||||
|
star_expr: '*' expr
|
||||||
|
expr: xor_expr ('|' xor_expr)*
|
||||||
|
xor_expr: and_expr ('^' and_expr)*
|
||||||
|
and_expr: shift_expr ('&' shift_expr)*
|
||||||
|
shift_expr: arith_expr (('<<'|'>>') arith_expr)*
|
||||||
|
arith_expr: term (('+'|'-') term)*
|
||||||
|
term: factor (('*'|'@'|'/'|'%'|'//') factor)*
|
||||||
|
factor: ('+'|'-'|'~') factor | power
|
||||||
|
power: ['await'] atom trailer* ['**' factor]
|
||||||
|
atom: ('(' [yield_expr|testlist_gexp] ')' |
|
||||||
|
'[' [listmaker] ']' |
|
||||||
|
'{' [dictsetmaker] '}' |
|
||||||
|
'`' testlist1 '`' |
|
||||||
|
NAME | NUMBER | STRING+ | '.' '.' '.')
|
||||||
|
listmaker: (test|star_expr) ( old_comp_for | (',' (test|star_expr))* [','] )
|
||||||
|
testlist_gexp: (test|star_expr) ( old_comp_for | (',' (test|star_expr))* [','] )
|
||||||
|
lambdef: 'lambda' [varargslist] ':' test
|
||||||
|
trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
|
||||||
|
subscriptlist: subscript (',' subscript)* [',']
|
||||||
|
subscript: test | [test] ':' [test] [sliceop]
|
||||||
|
sliceop: ':' [test]
|
||||||
|
exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
|
||||||
|
testlist: test (',' test)* [',']
|
||||||
|
dictsetmaker: ( ((test ':' test | '**' expr)
|
||||||
|
(comp_for | (',' (test ':' test | '**' expr))* [','])) |
|
||||||
|
((test | star_expr)
|
||||||
|
(comp_for | (',' (test | star_expr))* [','])) )
|
||||||
|
|
||||||
|
classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
|
||||||
|
|
||||||
|
arglist: argument (',' argument)* [',']
|
||||||
|
|
||||||
|
# "test '=' test" is really "keyword '=' test", but we have no such token.
|
||||||
|
# These need to be in a single rule to avoid grammar that is ambiguous
|
||||||
|
# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
|
||||||
|
# we explicitly match '*' here, too, to give it proper precedence.
|
||||||
|
# Illegal combinations and orderings are blocked in ast.c:
|
||||||
|
# multiple (test comp_for) arguments are blocked; keyword unpackings
|
||||||
|
# that precede iterable unpackings are blocked; etc.
|
||||||
|
argument: ( test [comp_for] |
|
||||||
|
test '=' test |
|
||||||
|
'**' expr |
|
||||||
|
star_expr )
|
||||||
|
|
||||||
|
comp_iter: comp_for | comp_if
|
||||||
|
comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter]
|
||||||
|
comp_if: 'if' old_test [comp_iter]
|
||||||
|
|
||||||
|
# As noted above, testlist_safe extends the syntax allowed in list
|
||||||
|
# comprehensions and generators. We can't use it indiscriminately in all
|
||||||
|
# derivations using a comp_for-like pattern because the testlist_safe derivation
|
||||||
|
# contains comma which clashes with trailing comma in arglist.
|
||||||
|
#
|
||||||
|
# This was an issue because the parser would not follow the correct derivation
|
||||||
|
# when parsing syntactically valid Python code. Since testlist_safe was created
|
||||||
|
# specifically to handle list comprehensions and generator expressions enclosed
|
||||||
|
# with parentheses, it's safe to only use it in those. That avoids the issue; we
|
||||||
|
# can parse code like set(x for x in [],).
|
||||||
|
#
|
||||||
|
# The syntax supported by this set of rules is not a valid Python 3 syntax,
|
||||||
|
# hence the prefix "old".
|
||||||
|
#
|
||||||
|
# See https://bugs.python.org/issue27494
|
||||||
|
old_comp_iter: old_comp_for | old_comp_if
|
||||||
|
old_comp_for: ['async'] 'for' exprlist 'in' testlist_safe [old_comp_iter]
|
||||||
|
old_comp_if: 'if' old_test [old_comp_iter]
|
||||||
|
|
||||||
|
testlist1: test (',' test)*
|
||||||
|
|
||||||
|
# not used in grammar, but may appear in "node" passed from Parser to Compiler
|
||||||
|
encoding_decl: NAME
|
||||||
|
|
||||||
|
yield_expr: 'yield' [yield_arg]
|
||||||
|
yield_arg: 'from' test | testlist
|
||||||
BIN
blib2to3/Grammar3.6.4.final.0.pickle
Normal file
BIN
blib2to3/Grammar3.6.4.final.0.pickle
Normal file
Binary file not shown.
28
blib2to3/PatternGrammar.txt
Normal file
28
blib2to3/PatternGrammar.txt
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
# Copyright 2006 Google, Inc. All Rights Reserved.
|
||||||
|
# Licensed to PSF under a Contributor Agreement.
|
||||||
|
|
||||||
|
# A grammar to describe tree matching patterns.
|
||||||
|
# Not shown here:
|
||||||
|
# - 'TOKEN' stands for any token (leaf node)
|
||||||
|
# - 'any' stands for any node (leaf or interior)
|
||||||
|
# With 'any' we can still specify the sub-structure.
|
||||||
|
|
||||||
|
# The start symbol is 'Matcher'.
|
||||||
|
|
||||||
|
Matcher: Alternatives ENDMARKER
|
||||||
|
|
||||||
|
Alternatives: Alternative ('|' Alternative)*
|
||||||
|
|
||||||
|
Alternative: (Unit | NegatedUnit)+
|
||||||
|
|
||||||
|
Unit: [NAME '='] ( STRING [Repeater]
|
||||||
|
| NAME [Details] [Repeater]
|
||||||
|
| '(' Alternatives ')' [Repeater]
|
||||||
|
| '[' Alternatives ']'
|
||||||
|
)
|
||||||
|
|
||||||
|
NegatedUnit: 'not' (STRING | NAME [Details] | '(' Alternatives ')')
|
||||||
|
|
||||||
|
Repeater: '*' | '+' | '{' NUMBER [',' NUMBER] '}'
|
||||||
|
|
||||||
|
Details: '<' Alternatives '>'
|
||||||
BIN
blib2to3/PatternGrammar3.6.4.final.0.pickle
Normal file
BIN
blib2to3/PatternGrammar3.6.4.final.0.pickle
Normal file
Binary file not shown.
7
blib2to3/README
Normal file
7
blib2to3/README
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
A subset of lib2to3 taken from Python 3.7.0b2.
|
||||||
|
Commit hash: 9c17e3a1987004b8bcfbe423953aad84493a7984
|
||||||
|
|
||||||
|
Reasons for forking:
|
||||||
|
- consistent handling of f-strings for users of Python < 3.6.2
|
||||||
|
- better ability to debug
|
||||||
|
- ability to Cythonize
|
||||||
1
blib2to3/__init__.py
Normal file
1
blib2to3/__init__.py
Normal file
@ -0,0 +1 @@
|
|||||||
|
#empty
|
||||||
1
blib2to3/__init__.pyi
Normal file
1
blib2to3/__init__.pyi
Normal file
@ -0,0 +1 @@
|
|||||||
|
# Stubs for lib2to3 (Python 3.6)
|
||||||
4
blib2to3/pgen2/__init__.py
Normal file
4
blib2to3/pgen2/__init__.py
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
|
||||||
|
# Licensed to PSF under a Contributor Agreement.
|
||||||
|
|
||||||
|
"""The pgen2 package."""
|
||||||
10
blib2to3/pgen2/__init__.pyi
Normal file
10
blib2to3/pgen2/__init__.pyi
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
# Stubs for lib2to3.pgen2 (Python 3.6)
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from typing import Text, Union
|
||||||
|
|
||||||
|
if sys.version_info >= (3, 6):
|
||||||
|
_Path = Union[Text, os.PathLike]
|
||||||
|
else:
|
||||||
|
_Path = Text
|
||||||
257
blib2to3/pgen2/conv.py
Normal file
257
blib2to3/pgen2/conv.py
Normal file
@ -0,0 +1,257 @@
|
|||||||
|
# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
|
||||||
|
# Licensed to PSF under a Contributor Agreement.
|
||||||
|
|
||||||
|
"""Convert graminit.[ch] spit out by pgen to Python code.
|
||||||
|
|
||||||
|
Pgen is the Python parser generator. It is useful to quickly create a
|
||||||
|
parser from a grammar file in Python's grammar notation. But I don't
|
||||||
|
want my parsers to be written in C (yet), so I'm translating the
|
||||||
|
parsing tables to Python data structures and writing a Python parse
|
||||||
|
engine.
|
||||||
|
|
||||||
|
Note that the token numbers are constants determined by the standard
|
||||||
|
Python tokenizer. The standard token module defines these numbers and
|
||||||
|
their names (the names are not used much). The token numbers are
|
||||||
|
hardcoded into the Python tokenizer and into pgen. A Python
|
||||||
|
implementation of the Python tokenizer is also available, in the
|
||||||
|
standard tokenize module.
|
||||||
|
|
||||||
|
On the other hand, symbol numbers (representing the grammar's
|
||||||
|
non-terminals) are assigned by pgen based on the actual grammar
|
||||||
|
input.
|
||||||
|
|
||||||
|
Note: this module is pretty much obsolete; the pgen module generates
|
||||||
|
equivalent grammar tables directly from the Grammar.txt input file
|
||||||
|
without having to invoke the Python pgen C program.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Python imports
|
||||||
|
import re
|
||||||
|
|
||||||
|
# Local imports
|
||||||
|
from pgen2 import grammar, token
|
||||||
|
|
||||||
|
|
||||||
|
class Converter(grammar.Grammar):
|
||||||
|
"""Grammar subclass that reads classic pgen output files.
|
||||||
|
|
||||||
|
The run() method reads the tables as produced by the pgen parser
|
||||||
|
generator, typically contained in two C files, graminit.h and
|
||||||
|
graminit.c. The other methods are for internal use only.
|
||||||
|
|
||||||
|
See the base class for more documentation.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
def run(self, graminit_h, graminit_c):
|
||||||
|
"""Load the grammar tables from the text files written by pgen."""
|
||||||
|
self.parse_graminit_h(graminit_h)
|
||||||
|
self.parse_graminit_c(graminit_c)
|
||||||
|
self.finish_off()
|
||||||
|
|
||||||
|
def parse_graminit_h(self, filename):
|
||||||
|
"""Parse the .h file written by pgen. (Internal)
|
||||||
|
|
||||||
|
This file is a sequence of #define statements defining the
|
||||||
|
nonterminals of the grammar as numbers. We build two tables
|
||||||
|
mapping the numbers to names and back.
|
||||||
|
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
f = open(filename)
|
||||||
|
except OSError as err:
|
||||||
|
print("Can't open %s: %s" % (filename, err))
|
||||||
|
return False
|
||||||
|
self.symbol2number = {}
|
||||||
|
self.number2symbol = {}
|
||||||
|
lineno = 0
|
||||||
|
for line in f:
|
||||||
|
lineno += 1
|
||||||
|
mo = re.match(r"^#define\s+(\w+)\s+(\d+)$", line)
|
||||||
|
if not mo and line.strip():
|
||||||
|
print("%s(%s): can't parse %s" % (filename, lineno,
|
||||||
|
line.strip()))
|
||||||
|
else:
|
||||||
|
symbol, number = mo.groups()
|
||||||
|
number = int(number)
|
||||||
|
assert symbol not in self.symbol2number
|
||||||
|
assert number not in self.number2symbol
|
||||||
|
self.symbol2number[symbol] = number
|
||||||
|
self.number2symbol[number] = symbol
|
||||||
|
return True
|
||||||
|
|
||||||
|
def parse_graminit_c(self, filename):
|
||||||
|
"""Parse the .c file written by pgen. (Internal)
|
||||||
|
|
||||||
|
The file looks as follows. The first two lines are always this:
|
||||||
|
|
||||||
|
#include "pgenheaders.h"
|
||||||
|
#include "grammar.h"
|
||||||
|
|
||||||
|
After that come four blocks:
|
||||||
|
|
||||||
|
1) one or more state definitions
|
||||||
|
2) a table defining dfas
|
||||||
|
3) a table defining labels
|
||||||
|
4) a struct defining the grammar
|
||||||
|
|
||||||
|
A state definition has the following form:
|
||||||
|
- one or more arc arrays, each of the form:
|
||||||
|
static arc arcs_<n>_<m>[<k>] = {
|
||||||
|
{<i>, <j>},
|
||||||
|
...
|
||||||
|
};
|
||||||
|
- followed by a state array, of the form:
|
||||||
|
static state states_<s>[<t>] = {
|
||||||
|
{<k>, arcs_<n>_<m>},
|
||||||
|
...
|
||||||
|
};
|
||||||
|
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
f = open(filename)
|
||||||
|
except OSError as err:
|
||||||
|
print("Can't open %s: %s" % (filename, err))
|
||||||
|
return False
|
||||||
|
# The code below essentially uses f's iterator-ness!
|
||||||
|
lineno = 0
|
||||||
|
|
||||||
|
# Expect the two #include lines
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
assert line == '#include "pgenheaders.h"\n', (lineno, line)
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
assert line == '#include "grammar.h"\n', (lineno, line)
|
||||||
|
|
||||||
|
# Parse the state definitions
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
allarcs = {}
|
||||||
|
states = []
|
||||||
|
while line.startswith("static arc "):
|
||||||
|
while line.startswith("static arc "):
|
||||||
|
mo = re.match(r"static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$",
|
||||||
|
line)
|
||||||
|
assert mo, (lineno, line)
|
||||||
|
n, m, k = list(map(int, mo.groups()))
|
||||||
|
arcs = []
|
||||||
|
for _ in range(k):
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
mo = re.match(r"\s+{(\d+), (\d+)},$", line)
|
||||||
|
assert mo, (lineno, line)
|
||||||
|
i, j = list(map(int, mo.groups()))
|
||||||
|
arcs.append((i, j))
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
assert line == "};\n", (lineno, line)
|
||||||
|
allarcs[(n, m)] = arcs
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
mo = re.match(r"static state states_(\d+)\[(\d+)\] = {$", line)
|
||||||
|
assert mo, (lineno, line)
|
||||||
|
s, t = list(map(int, mo.groups()))
|
||||||
|
assert s == len(states), (lineno, line)
|
||||||
|
state = []
|
||||||
|
for _ in range(t):
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
mo = re.match(r"\s+{(\d+), arcs_(\d+)_(\d+)},$", line)
|
||||||
|
assert mo, (lineno, line)
|
||||||
|
k, n, m = list(map(int, mo.groups()))
|
||||||
|
arcs = allarcs[n, m]
|
||||||
|
assert k == len(arcs), (lineno, line)
|
||||||
|
state.append(arcs)
|
||||||
|
states.append(state)
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
assert line == "};\n", (lineno, line)
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
self.states = states
|
||||||
|
|
||||||
|
# Parse the dfas
|
||||||
|
dfas = {}
|
||||||
|
mo = re.match(r"static dfa dfas\[(\d+)\] = {$", line)
|
||||||
|
assert mo, (lineno, line)
|
||||||
|
ndfas = int(mo.group(1))
|
||||||
|
for i in range(ndfas):
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
mo = re.match(r'\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$',
|
||||||
|
line)
|
||||||
|
assert mo, (lineno, line)
|
||||||
|
symbol = mo.group(2)
|
||||||
|
number, x, y, z = list(map(int, mo.group(1, 3, 4, 5)))
|
||||||
|
assert self.symbol2number[symbol] == number, (lineno, line)
|
||||||
|
assert self.number2symbol[number] == symbol, (lineno, line)
|
||||||
|
assert x == 0, (lineno, line)
|
||||||
|
state = states[z]
|
||||||
|
assert y == len(state), (lineno, line)
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
mo = re.match(r'\s+("(?:\\\d\d\d)*")},$', line)
|
||||||
|
assert mo, (lineno, line)
|
||||||
|
first = {}
|
||||||
|
rawbitset = eval(mo.group(1))
|
||||||
|
for i, c in enumerate(rawbitset):
|
||||||
|
byte = ord(c)
|
||||||
|
for j in range(8):
|
||||||
|
if byte & (1<<j):
|
||||||
|
first[i*8 + j] = 1
|
||||||
|
dfas[number] = (state, first)
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
assert line == "};\n", (lineno, line)
|
||||||
|
self.dfas = dfas
|
||||||
|
|
||||||
|
# Parse the labels
|
||||||
|
labels = []
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
mo = re.match(r"static label labels\[(\d+)\] = {$", line)
|
||||||
|
assert mo, (lineno, line)
|
||||||
|
nlabels = int(mo.group(1))
|
||||||
|
for i in range(nlabels):
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
mo = re.match(r'\s+{(\d+), (0|"\w+")},$', line)
|
||||||
|
assert mo, (lineno, line)
|
||||||
|
x, y = mo.groups()
|
||||||
|
x = int(x)
|
||||||
|
if y == "0":
|
||||||
|
y = None
|
||||||
|
else:
|
||||||
|
y = eval(y)
|
||||||
|
labels.append((x, y))
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
assert line == "};\n", (lineno, line)
|
||||||
|
self.labels = labels
|
||||||
|
|
||||||
|
# Parse the grammar struct
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
assert line == "grammar _PyParser_Grammar = {\n", (lineno, line)
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
mo = re.match(r"\s+(\d+),$", line)
|
||||||
|
assert mo, (lineno, line)
|
||||||
|
ndfas = int(mo.group(1))
|
||||||
|
assert ndfas == len(self.dfas)
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
assert line == "\tdfas,\n", (lineno, line)
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
mo = re.match(r"\s+{(\d+), labels},$", line)
|
||||||
|
assert mo, (lineno, line)
|
||||||
|
nlabels = int(mo.group(1))
|
||||||
|
assert nlabels == len(self.labels), (lineno, line)
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
mo = re.match(r"\s+(\d+)$", line)
|
||||||
|
assert mo, (lineno, line)
|
||||||
|
start = int(mo.group(1))
|
||||||
|
assert start in self.number2symbol, (lineno, line)
|
||||||
|
self.start = start
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
assert line == "};\n", (lineno, line)
|
||||||
|
try:
|
||||||
|
lineno, line = lineno+1, next(f)
|
||||||
|
except StopIteration:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
assert 0, (lineno, line)
|
||||||
|
|
||||||
|
def finish_off(self):
|
||||||
|
"""Create additional useful structures. (Internal)."""
|
||||||
|
self.keywords = {} # map from keyword strings to arc labels
|
||||||
|
self.tokens = {} # map from numeric token values to arc labels
|
||||||
|
for ilabel, (type, value) in enumerate(self.labels):
|
||||||
|
if type == token.NAME and value is not None:
|
||||||
|
self.keywords[value] = ilabel
|
||||||
|
elif value is None:
|
||||||
|
self.tokens[type] = ilabel
|
||||||
178
blib2to3/pgen2/driver.py
Normal file
178
blib2to3/pgen2/driver.py
Normal file
@ -0,0 +1,178 @@
|
|||||||
|
# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
|
||||||
|
# Licensed to PSF under a Contributor Agreement.
|
||||||
|
|
||||||
|
# Modifications:
|
||||||
|
# Copyright 2006 Google, Inc. All Rights Reserved.
|
||||||
|
# Licensed to PSF under a Contributor Agreement.
|
||||||
|
|
||||||
|
"""Parser driver.
|
||||||
|
|
||||||
|
This provides a high-level interface to parse a file into a syntax tree.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
__author__ = "Guido van Rossum <guido@python.org>"
|
||||||
|
|
||||||
|
__all__ = ["Driver", "load_grammar"]
|
||||||
|
|
||||||
|
# Python imports
|
||||||
|
import codecs
|
||||||
|
import io
|
||||||
|
import os
|
||||||
|
import logging
|
||||||
|
import pkgutil
|
||||||
|
import sys
|
||||||
|
|
||||||
|
# Pgen imports
|
||||||
|
from . import grammar, parse, token, tokenize, pgen
|
||||||
|
|
||||||
|
|
||||||
|
class Driver(object):
|
||||||
|
|
||||||
|
def __init__(self, grammar, convert=None, logger=None):
|
||||||
|
self.grammar = grammar
|
||||||
|
if logger is None:
|
||||||
|
logger = logging.getLogger()
|
||||||
|
self.logger = logger
|
||||||
|
self.convert = convert
|
||||||
|
|
||||||
|
def parse_tokens(self, tokens, debug=False):
|
||||||
|
"""Parse a series of tokens and return the syntax tree."""
|
||||||
|
# XXX Move the prefix computation into a wrapper around tokenize.
|
||||||
|
p = parse.Parser(self.grammar, self.convert)
|
||||||
|
p.setup()
|
||||||
|
lineno = 1
|
||||||
|
column = 0
|
||||||
|
type = value = start = end = line_text = None
|
||||||
|
prefix = ""
|
||||||
|
for quintuple in tokens:
|
||||||
|
type, value, start, end, line_text = quintuple
|
||||||
|
if start != (lineno, column):
|
||||||
|
assert (lineno, column) <= start, ((lineno, column), start)
|
||||||
|
s_lineno, s_column = start
|
||||||
|
if lineno < s_lineno:
|
||||||
|
prefix += "\n" * (s_lineno - lineno)
|
||||||
|
lineno = s_lineno
|
||||||
|
column = 0
|
||||||
|
if column < s_column:
|
||||||
|
prefix += line_text[column:s_column]
|
||||||
|
column = s_column
|
||||||
|
if type in (tokenize.COMMENT, tokenize.NL):
|
||||||
|
prefix += value
|
||||||
|
lineno, column = end
|
||||||
|
if value.endswith("\n"):
|
||||||
|
lineno += 1
|
||||||
|
column = 0
|
||||||
|
continue
|
||||||
|
if type == token.OP:
|
||||||
|
type = grammar.opmap[value]
|
||||||
|
if debug:
|
||||||
|
self.logger.debug("%s %r (prefix=%r)",
|
||||||
|
token.tok_name[type], value, prefix)
|
||||||
|
if p.addtoken(type, value, (prefix, start)):
|
||||||
|
if debug:
|
||||||
|
self.logger.debug("Stop.")
|
||||||
|
break
|
||||||
|
prefix = ""
|
||||||
|
lineno, column = end
|
||||||
|
if value.endswith("\n"):
|
||||||
|
lineno += 1
|
||||||
|
column = 0
|
||||||
|
else:
|
||||||
|
# We never broke out -- EOF is too soon (how can this happen???)
|
||||||
|
raise parse.ParseError("incomplete input",
|
||||||
|
type, value, (prefix, start))
|
||||||
|
return p.rootnode
|
||||||
|
|
||||||
|
def parse_stream_raw(self, stream, debug=False):
|
||||||
|
"""Parse a stream and return the syntax tree."""
|
||||||
|
tokens = tokenize.generate_tokens(stream.readline)
|
||||||
|
return self.parse_tokens(tokens, debug)
|
||||||
|
|
||||||
|
def parse_stream(self, stream, debug=False):
|
||||||
|
"""Parse a stream and return the syntax tree."""
|
||||||
|
return self.parse_stream_raw(stream, debug)
|
||||||
|
|
||||||
|
def parse_file(self, filename, encoding=None, debug=False):
|
||||||
|
"""Parse a file and return the syntax tree."""
|
||||||
|
with io.open(filename, "r", encoding=encoding) as stream:
|
||||||
|
return self.parse_stream(stream, debug)
|
||||||
|
|
||||||
|
def parse_string(self, text, debug=False):
|
||||||
|
"""Parse a string and return the syntax tree."""
|
||||||
|
tokens = tokenize.generate_tokens(io.StringIO(text).readline)
|
||||||
|
return self.parse_tokens(tokens, debug)
|
||||||
|
|
||||||
|
|
||||||
|
def _generate_pickle_name(gt):
|
||||||
|
head, tail = os.path.splitext(gt)
|
||||||
|
if tail == ".txt":
|
||||||
|
tail = ""
|
||||||
|
return head + tail + ".".join(map(str, sys.version_info)) + ".pickle"
|
||||||
|
|
||||||
|
|
||||||
|
def load_grammar(gt="Grammar.txt", gp=None,
|
||||||
|
save=True, force=False, logger=None):
|
||||||
|
"""Load the grammar (maybe from a pickle)."""
|
||||||
|
if logger is None:
|
||||||
|
logger = logging.getLogger()
|
||||||
|
gp = _generate_pickle_name(gt) if gp is None else gp
|
||||||
|
if force or not _newer(gp, gt):
|
||||||
|
logger.info("Generating grammar tables from %s", gt)
|
||||||
|
g = pgen.generate_grammar(gt)
|
||||||
|
if save:
|
||||||
|
logger.info("Writing grammar tables to %s", gp)
|
||||||
|
try:
|
||||||
|
g.dump(gp)
|
||||||
|
except OSError as e:
|
||||||
|
logger.info("Writing failed: %s", e)
|
||||||
|
else:
|
||||||
|
g = grammar.Grammar()
|
||||||
|
g.load(gp)
|
||||||
|
return g
|
||||||
|
|
||||||
|
|
||||||
|
def _newer(a, b):
|
||||||
|
"""Inquire whether file a was written since file b."""
|
||||||
|
if not os.path.exists(a):
|
||||||
|
return False
|
||||||
|
if not os.path.exists(b):
|
||||||
|
return True
|
||||||
|
return os.path.getmtime(a) >= os.path.getmtime(b)
|
||||||
|
|
||||||
|
|
||||||
|
def load_packaged_grammar(package, grammar_source):
|
||||||
|
"""Normally, loads a pickled grammar by doing
|
||||||
|
pkgutil.get_data(package, pickled_grammar)
|
||||||
|
where *pickled_grammar* is computed from *grammar_source* by adding the
|
||||||
|
Python version and using a ``.pickle`` extension.
|
||||||
|
|
||||||
|
However, if *grammar_source* is an extant file, load_grammar(grammar_source)
|
||||||
|
is called instead. This facilitates using a packaged grammar file when needed
|
||||||
|
but preserves load_grammar's automatic regeneration behavior when possible.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if os.path.isfile(grammar_source):
|
||||||
|
return load_grammar(grammar_source)
|
||||||
|
pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
|
||||||
|
data = pkgutil.get_data(package, pickled_name)
|
||||||
|
g = grammar.Grammar()
|
||||||
|
g.loads(data)
|
||||||
|
return g
|
||||||
|
|
||||||
|
|
||||||
|
def main(*args):
|
||||||
|
"""Main program, when run as a script: produce grammar pickle files.
|
||||||
|
|
||||||
|
Calls load_grammar for each argument, a path to a grammar text file.
|
||||||
|
"""
|
||||||
|
if not args:
|
||||||
|
args = sys.argv[1:]
|
||||||
|
logging.basicConfig(level=logging.INFO, stream=sys.stdout,
|
||||||
|
format='%(message)s')
|
||||||
|
for gt in args:
|
||||||
|
load_grammar(gt, save=True, force=True)
|
||||||
|
return True
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(int(not main()))
|
||||||
24
blib2to3/pgen2/driver.pyi
Normal file
24
blib2to3/pgen2/driver.pyi
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
# Stubs for lib2to3.pgen2.driver (Python 3.6)
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from typing import Any, Callable, IO, Iterable, List, Optional, Text, Tuple, Union
|
||||||
|
|
||||||
|
from logging import Logger
|
||||||
|
from blib2to3.pytree import _Convert, _NL
|
||||||
|
from blib2to3.pgen2 import _Path
|
||||||
|
from blib2to3.pgen2.grammar import Grammar
|
||||||
|
|
||||||
|
|
||||||
|
class Driver:
|
||||||
|
grammar: Grammar
|
||||||
|
logger: Logger
|
||||||
|
convert: _Convert
|
||||||
|
def __init__(self, grammar: Grammar, convert: Optional[_Convert] = ..., logger: Optional[Logger] = ...) -> None: ...
|
||||||
|
def parse_tokens(self, tokens: Iterable[Any], debug: bool = ...) -> _NL: ...
|
||||||
|
def parse_stream_raw(self, stream: IO[Text], debug: bool = ...) -> _NL: ...
|
||||||
|
def parse_stream(self, stream: IO[Text], debug: bool = ...) -> _NL: ...
|
||||||
|
def parse_file(self, filename: _Path, encoding: Optional[Text] = ..., debug: bool = ...) -> _NL: ...
|
||||||
|
def parse_string(self, text: Text, debug: bool = ...) -> _NL: ...
|
||||||
|
|
||||||
|
def load_grammar(gt: Text = ..., gp: Optional[Text] = ..., save: bool = ..., force: bool = ..., logger: Optional[Logger] = ...) -> Grammar: ...
|
||||||
211
blib2to3/pgen2/grammar.py
Normal file
211
blib2to3/pgen2/grammar.py
Normal file
@ -0,0 +1,211 @@
|
|||||||
|
# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
|
||||||
|
# Licensed to PSF under a Contributor Agreement.
|
||||||
|
|
||||||
|
"""This module defines the data structures used to represent a grammar.
|
||||||
|
|
||||||
|
These are a bit arcane because they are derived from the data
|
||||||
|
structures used by Python's 'pgen' parser generator.
|
||||||
|
|
||||||
|
There's also a table here mapping operators to their names in the
|
||||||
|
token module; the Python tokenize module reports all operators as the
|
||||||
|
fallback token code OP, but the parser needs the actual token code.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Python imports
|
||||||
|
import collections
|
||||||
|
import pickle
|
||||||
|
|
||||||
|
# Local imports
|
||||||
|
from . import token
|
||||||
|
|
||||||
|
|
||||||
|
class Grammar(object):
|
||||||
|
"""Pgen parsing tables conversion class.
|
||||||
|
|
||||||
|
Once initialized, this class supplies the grammar tables for the
|
||||||
|
parsing engine implemented by parse.py. The parsing engine
|
||||||
|
accesses the instance variables directly. The class here does not
|
||||||
|
provide initialization of the tables; several subclasses exist to
|
||||||
|
do this (see the conv and pgen modules).
|
||||||
|
|
||||||
|
The load() method reads the tables from a pickle file, which is
|
||||||
|
much faster than the other ways offered by subclasses. The pickle
|
||||||
|
file is written by calling dump() (after loading the grammar
|
||||||
|
tables using a subclass). The report() method prints a readable
|
||||||
|
representation of the tables to stdout, for debugging.
|
||||||
|
|
||||||
|
The instance variables are as follows:
|
||||||
|
|
||||||
|
symbol2number -- a dict mapping symbol names to numbers. Symbol
|
||||||
|
numbers are always 256 or higher, to distinguish
|
||||||
|
them from token numbers, which are between 0 and
|
||||||
|
255 (inclusive).
|
||||||
|
|
||||||
|
number2symbol -- a dict mapping numbers to symbol names;
|
||||||
|
these two are each other's inverse.
|
||||||
|
|
||||||
|
states -- a list of DFAs, where each DFA is a list of
|
||||||
|
states, each state is a list of arcs, and each
|
||||||
|
arc is a (i, j) pair where i is a label and j is
|
||||||
|
a state number. The DFA number is the index into
|
||||||
|
this list. (This name is slightly confusing.)
|
||||||
|
Final states are represented by a special arc of
|
||||||
|
the form (0, j) where j is its own state number.
|
||||||
|
|
||||||
|
dfas -- a dict mapping symbol numbers to (DFA, first)
|
||||||
|
pairs, where DFA is an item from the states list
|
||||||
|
above, and first is a set of tokens that can
|
||||||
|
begin this grammar rule (represented by a dict
|
||||||
|
whose values are always 1).
|
||||||
|
|
||||||
|
labels -- a list of (x, y) pairs where x is either a token
|
||||||
|
number or a symbol number, and y is either None
|
||||||
|
or a string; the strings are keywords. The label
|
||||||
|
number is the index in this list; label numbers
|
||||||
|
are used to mark state transitions (arcs) in the
|
||||||
|
DFAs.
|
||||||
|
|
||||||
|
start -- the number of the grammar's start symbol.
|
||||||
|
|
||||||
|
keywords -- a dict mapping keyword strings to arc labels.
|
||||||
|
|
||||||
|
tokens -- a dict mapping token numbers to arc labels.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.symbol2number = {}
|
||||||
|
self.number2symbol = {}
|
||||||
|
self.states = []
|
||||||
|
self.dfas = {}
|
||||||
|
self.labels = [(0, "EMPTY")]
|
||||||
|
self.keywords = {}
|
||||||
|
self.tokens = {}
|
||||||
|
self.symbol2label = {}
|
||||||
|
self.start = 256
|
||||||
|
|
||||||
|
def dump(self, filename):
|
||||||
|
"""Dump the grammar tables to a pickle file.
|
||||||
|
|
||||||
|
dump() recursively changes all dict to OrderedDict, so the pickled file
|
||||||
|
is not exactly the same as what was passed in to dump(). load() uses the
|
||||||
|
pickled file to create the tables, but only changes OrderedDict to dict
|
||||||
|
at the top level; it does not recursively change OrderedDict to dict.
|
||||||
|
So, the loaded tables are different from the original tables that were
|
||||||
|
passed to load() in that some of the OrderedDict (from the pickled file)
|
||||||
|
are not changed back to dict. For parsing, this has no effect on
|
||||||
|
performance because OrderedDict uses dict's __getitem__ with nothing in
|
||||||
|
between.
|
||||||
|
"""
|
||||||
|
with open(filename, "wb") as f:
|
||||||
|
d = _make_deterministic(self.__dict__)
|
||||||
|
pickle.dump(d, f, 2)
|
||||||
|
|
||||||
|
def load(self, filename):
|
||||||
|
"""Load the grammar tables from a pickle file."""
|
||||||
|
with open(filename, "rb") as f:
|
||||||
|
d = pickle.load(f)
|
||||||
|
self.__dict__.update(d)
|
||||||
|
|
||||||
|
def loads(self, pkl):
|
||||||
|
"""Load the grammar tables from a pickle bytes object."""
|
||||||
|
self.__dict__.update(pickle.loads(pkl))
|
||||||
|
|
||||||
|
def copy(self):
|
||||||
|
"""
|
||||||
|
Copy the grammar.
|
||||||
|
"""
|
||||||
|
new = self.__class__()
|
||||||
|
for dict_attr in ("symbol2number", "number2symbol", "dfas", "keywords",
|
||||||
|
"tokens", "symbol2label"):
|
||||||
|
setattr(new, dict_attr, getattr(self, dict_attr).copy())
|
||||||
|
new.labels = self.labels[:]
|
||||||
|
new.states = self.states[:]
|
||||||
|
new.start = self.start
|
||||||
|
return new
|
||||||
|
|
||||||
|
def report(self):
|
||||||
|
"""Dump the grammar tables to standard output, for debugging."""
|
||||||
|
from pprint import pprint
|
||||||
|
print("s2n")
|
||||||
|
pprint(self.symbol2number)
|
||||||
|
print("n2s")
|
||||||
|
pprint(self.number2symbol)
|
||||||
|
print("states")
|
||||||
|
pprint(self.states)
|
||||||
|
print("dfas")
|
||||||
|
pprint(self.dfas)
|
||||||
|
print("labels")
|
||||||
|
pprint(self.labels)
|
||||||
|
print("start", self.start)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_deterministic(top):
|
||||||
|
if isinstance(top, dict):
|
||||||
|
return collections.OrderedDict(
|
||||||
|
sorted(((k, _make_deterministic(v)) for k, v in top.items())))
|
||||||
|
if isinstance(top, list):
|
||||||
|
return [_make_deterministic(e) for e in top]
|
||||||
|
if isinstance(top, tuple):
|
||||||
|
return tuple(_make_deterministic(e) for e in top)
|
||||||
|
return top
|
||||||
|
|
||||||
|
|
||||||
|
# Map from operator to number (since tokenize doesn't do this)
|
||||||
|
|
||||||
|
opmap_raw = """
|
||||||
|
( LPAR
|
||||||
|
) RPAR
|
||||||
|
[ LSQB
|
||||||
|
] RSQB
|
||||||
|
: COLON
|
||||||
|
, COMMA
|
||||||
|
; SEMI
|
||||||
|
+ PLUS
|
||||||
|
- MINUS
|
||||||
|
* STAR
|
||||||
|
/ SLASH
|
||||||
|
| VBAR
|
||||||
|
& AMPER
|
||||||
|
< LESS
|
||||||
|
> GREATER
|
||||||
|
= EQUAL
|
||||||
|
. DOT
|
||||||
|
% PERCENT
|
||||||
|
` BACKQUOTE
|
||||||
|
{ LBRACE
|
||||||
|
} RBRACE
|
||||||
|
@ AT
|
||||||
|
@= ATEQUAL
|
||||||
|
== EQEQUAL
|
||||||
|
!= NOTEQUAL
|
||||||
|
<> NOTEQUAL
|
||||||
|
<= LESSEQUAL
|
||||||
|
>= GREATEREQUAL
|
||||||
|
~ TILDE
|
||||||
|
^ CIRCUMFLEX
|
||||||
|
<< LEFTSHIFT
|
||||||
|
>> RIGHTSHIFT
|
||||||
|
** DOUBLESTAR
|
||||||
|
+= PLUSEQUAL
|
||||||
|
-= MINEQUAL
|
||||||
|
*= STAREQUAL
|
||||||
|
/= SLASHEQUAL
|
||||||
|
%= PERCENTEQUAL
|
||||||
|
&= AMPEREQUAL
|
||||||
|
|= VBAREQUAL
|
||||||
|
^= CIRCUMFLEXEQUAL
|
||||||
|
<<= LEFTSHIFTEQUAL
|
||||||
|
>>= RIGHTSHIFTEQUAL
|
||||||
|
**= DOUBLESTAREQUAL
|
||||||
|
// DOUBLESLASH
|
||||||
|
//= DOUBLESLASHEQUAL
|
||||||
|
-> RARROW
|
||||||
|
"""
|
||||||
|
|
||||||
|
opmap = {}
|
||||||
|
for line in opmap_raw.splitlines():
|
||||||
|
if line:
|
||||||
|
op, name = line.split()
|
||||||
|
opmap[op] = getattr(token, name)
|
||||||
29
blib2to3/pgen2/grammar.pyi
Normal file
29
blib2to3/pgen2/grammar.pyi
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
# Stubs for lib2to3.pgen2.grammar (Python 3.6)
|
||||||
|
|
||||||
|
from blib2to3.pgen2 import _Path
|
||||||
|
|
||||||
|
from typing import Any, Dict, List, Optional, Text, Tuple, TypeVar
|
||||||
|
|
||||||
|
_P = TypeVar('_P')
|
||||||
|
_Label = Tuple[int, Optional[Text]]
|
||||||
|
_DFA = List[List[Tuple[int, int]]]
|
||||||
|
_DFAS = Tuple[_DFA, Dict[int, int]]
|
||||||
|
|
||||||
|
class Grammar:
|
||||||
|
symbol2number: Dict[Text, int]
|
||||||
|
number2symbol: Dict[int, Text]
|
||||||
|
states: List[_DFA]
|
||||||
|
dfas: Dict[int, _DFAS]
|
||||||
|
labels: List[_Label]
|
||||||
|
keywords: Dict[Text, int]
|
||||||
|
tokens: Dict[int, int]
|
||||||
|
symbol2label: Dict[Text, int]
|
||||||
|
start: int
|
||||||
|
def __init__(self) -> None: ...
|
||||||
|
def dump(self, filename: _Path) -> None: ...
|
||||||
|
def load(self, filename: _Path) -> None: ...
|
||||||
|
def copy(self: _P) -> _P: ...
|
||||||
|
def report(self) -> None: ...
|
||||||
|
|
||||||
|
opmap_raw: Text
|
||||||
|
opmap: Dict[Text, Text]
|
||||||
60
blib2to3/pgen2/literals.py
Normal file
60
blib2to3/pgen2/literals.py
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
|
||||||
|
# Licensed to PSF under a Contributor Agreement.
|
||||||
|
|
||||||
|
"""Safely evaluate Python string literals without using eval()."""
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
simple_escapes = {"a": "\a",
|
||||||
|
"b": "\b",
|
||||||
|
"f": "\f",
|
||||||
|
"n": "\n",
|
||||||
|
"r": "\r",
|
||||||
|
"t": "\t",
|
||||||
|
"v": "\v",
|
||||||
|
"'": "'",
|
||||||
|
'"': '"',
|
||||||
|
"\\": "\\"}
|
||||||
|
|
||||||
|
def escape(m):
|
||||||
|
all, tail = m.group(0, 1)
|
||||||
|
assert all.startswith("\\")
|
||||||
|
esc = simple_escapes.get(tail)
|
||||||
|
if esc is not None:
|
||||||
|
return esc
|
||||||
|
if tail.startswith("x"):
|
||||||
|
hexes = tail[1:]
|
||||||
|
if len(hexes) < 2:
|
||||||
|
raise ValueError("invalid hex string escape ('\\%s')" % tail)
|
||||||
|
try:
|
||||||
|
i = int(hexes, 16)
|
||||||
|
except ValueError:
|
||||||
|
raise ValueError("invalid hex string escape ('\\%s')" % tail) from None
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
i = int(tail, 8)
|
||||||
|
except ValueError:
|
||||||
|
raise ValueError("invalid octal string escape ('\\%s')" % tail) from None
|
||||||
|
return chr(i)
|
||||||
|
|
||||||
|
def evalString(s):
|
||||||
|
assert s.startswith("'") or s.startswith('"'), repr(s[:1])
|
||||||
|
q = s[0]
|
||||||
|
if s[:3] == q*3:
|
||||||
|
q = q*3
|
||||||
|
assert s.endswith(q), repr(s[-len(q):])
|
||||||
|
assert len(s) >= 2*len(q)
|
||||||
|
s = s[len(q):-len(q)]
|
||||||
|
return re.sub(r"\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3})", escape, s)
|
||||||
|
|
||||||
|
def test():
|
||||||
|
for i in range(256):
|
||||||
|
c = chr(i)
|
||||||
|
s = repr(c)
|
||||||
|
e = evalString(s)
|
||||||
|
if e != c:
|
||||||
|
print(i, c, s, e)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
test()
|
||||||
9
blib2to3/pgen2/literals.pyi
Normal file
9
blib2to3/pgen2/literals.pyi
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
# Stubs for lib2to3.pgen2.literals (Python 3.6)
|
||||||
|
|
||||||
|
from typing import Dict, Match, Text
|
||||||
|
|
||||||
|
simple_escapes: Dict[Text, Text]
|
||||||
|
|
||||||
|
def escape(m: Match) -> Text: ...
|
||||||
|
def evalString(s: Text) -> Text: ...
|
||||||
|
def test() -> None: ...
|
||||||
201
blib2to3/pgen2/parse.py
Normal file
201
blib2to3/pgen2/parse.py
Normal file
@ -0,0 +1,201 @@
|
|||||||
|
# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
|
||||||
|
# Licensed to PSF under a Contributor Agreement.
|
||||||
|
|
||||||
|
"""Parser engine for the grammar tables generated by pgen.
|
||||||
|
|
||||||
|
The grammar table must be loaded first.
|
||||||
|
|
||||||
|
See Parser/parser.c in the Python distribution for additional info on
|
||||||
|
how this parsing engine works.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Local imports
|
||||||
|
from . import token
|
||||||
|
|
||||||
|
class ParseError(Exception):
|
||||||
|
"""Exception to signal the parser is stuck."""
|
||||||
|
|
||||||
|
def __init__(self, msg, type, value, context):
|
||||||
|
Exception.__init__(self, "%s: type=%r, value=%r, context=%r" %
|
||||||
|
(msg, type, value, context))
|
||||||
|
self.msg = msg
|
||||||
|
self.type = type
|
||||||
|
self.value = value
|
||||||
|
self.context = context
|
||||||
|
|
||||||
|
class Parser(object):
|
||||||
|
"""Parser engine.
|
||||||
|
|
||||||
|
The proper usage sequence is:
|
||||||
|
|
||||||
|
p = Parser(grammar, [converter]) # create instance
|
||||||
|
p.setup([start]) # prepare for parsing
|
||||||
|
<for each input token>:
|
||||||
|
if p.addtoken(...): # parse a token; may raise ParseError
|
||||||
|
break
|
||||||
|
root = p.rootnode # root of abstract syntax tree
|
||||||
|
|
||||||
|
A Parser instance may be reused by calling setup() repeatedly.
|
||||||
|
|
||||||
|
A Parser instance contains state pertaining to the current token
|
||||||
|
sequence, and should not be used concurrently by different threads
|
||||||
|
to parse separate token sequences.
|
||||||
|
|
||||||
|
See driver.py for how to get input tokens by tokenizing a file or
|
||||||
|
string.
|
||||||
|
|
||||||
|
Parsing is complete when addtoken() returns True; the root of the
|
||||||
|
abstract syntax tree can then be retrieved from the rootnode
|
||||||
|
instance variable. When a syntax error occurs, addtoken() raises
|
||||||
|
the ParseError exception. There is no error recovery; the parser
|
||||||
|
cannot be used after a syntax error was reported (but it can be
|
||||||
|
reinitialized by calling setup()).
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, grammar, convert=None):
|
||||||
|
"""Constructor.
|
||||||
|
|
||||||
|
The grammar argument is a grammar.Grammar instance; see the
|
||||||
|
grammar module for more information.
|
||||||
|
|
||||||
|
The parser is not ready yet for parsing; you must call the
|
||||||
|
setup() method to get it started.
|
||||||
|
|
||||||
|
The optional convert argument is a function mapping concrete
|
||||||
|
syntax tree nodes to abstract syntax tree nodes. If not
|
||||||
|
given, no conversion is done and the syntax tree produced is
|
||||||
|
the concrete syntax tree. If given, it must be a function of
|
||||||
|
two arguments, the first being the grammar (a grammar.Grammar
|
||||||
|
instance), and the second being the concrete syntax tree node
|
||||||
|
to be converted. The syntax tree is converted from the bottom
|
||||||
|
up.
|
||||||
|
|
||||||
|
A concrete syntax tree node is a (type, value, context, nodes)
|
||||||
|
tuple, where type is the node type (a token or symbol number),
|
||||||
|
value is None for symbols and a string for tokens, context is
|
||||||
|
None or an opaque value used for error reporting (typically a
|
||||||
|
(lineno, offset) pair), and nodes is a list of children for
|
||||||
|
symbols, and None for tokens.
|
||||||
|
|
||||||
|
An abstract syntax tree node may be anything; this is entirely
|
||||||
|
up to the converter function.
|
||||||
|
|
||||||
|
"""
|
||||||
|
self.grammar = grammar
|
||||||
|
self.convert = convert or (lambda grammar, node: node)
|
||||||
|
|
||||||
|
def setup(self, start=None):
|
||||||
|
"""Prepare for parsing.
|
||||||
|
|
||||||
|
This *must* be called before starting to parse.
|
||||||
|
|
||||||
|
The optional argument is an alternative start symbol; it
|
||||||
|
defaults to the grammar's start symbol.
|
||||||
|
|
||||||
|
You can use a Parser instance to parse any number of programs;
|
||||||
|
each time you call setup() the parser is reset to an initial
|
||||||
|
state determined by the (implicit or explicit) start symbol.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if start is None:
|
||||||
|
start = self.grammar.start
|
||||||
|
# Each stack entry is a tuple: (dfa, state, node).
|
||||||
|
# A node is a tuple: (type, value, context, children),
|
||||||
|
# where children is a list of nodes or None, and context may be None.
|
||||||
|
newnode = (start, None, None, [])
|
||||||
|
stackentry = (self.grammar.dfas[start], 0, newnode)
|
||||||
|
self.stack = [stackentry]
|
||||||
|
self.rootnode = None
|
||||||
|
self.used_names = set() # Aliased to self.rootnode.used_names in pop()
|
||||||
|
|
||||||
|
def addtoken(self, type, value, context):
|
||||||
|
"""Add a token; return True iff this is the end of the program."""
|
||||||
|
# Map from token to label
|
||||||
|
ilabel = self.classify(type, value, context)
|
||||||
|
# Loop until the token is shifted; may raise exceptions
|
||||||
|
while True:
|
||||||
|
dfa, state, node = self.stack[-1]
|
||||||
|
states, first = dfa
|
||||||
|
arcs = states[state]
|
||||||
|
# Look for a state with this label
|
||||||
|
for i, newstate in arcs:
|
||||||
|
t, v = self.grammar.labels[i]
|
||||||
|
if ilabel == i:
|
||||||
|
# Look it up in the list of labels
|
||||||
|
assert t < 256
|
||||||
|
# Shift a token; we're done with it
|
||||||
|
self.shift(type, value, newstate, context)
|
||||||
|
# Pop while we are in an accept-only state
|
||||||
|
state = newstate
|
||||||
|
while states[state] == [(0, state)]:
|
||||||
|
self.pop()
|
||||||
|
if not self.stack:
|
||||||
|
# Done parsing!
|
||||||
|
return True
|
||||||
|
dfa, state, node = self.stack[-1]
|
||||||
|
states, first = dfa
|
||||||
|
# Done with this token
|
||||||
|
return False
|
||||||
|
elif t >= 256:
|
||||||
|
# See if it's a symbol and if we're in its first set
|
||||||
|
itsdfa = self.grammar.dfas[t]
|
||||||
|
itsstates, itsfirst = itsdfa
|
||||||
|
if ilabel in itsfirst:
|
||||||
|
# Push a symbol
|
||||||
|
self.push(t, self.grammar.dfas[t], newstate, context)
|
||||||
|
break # To continue the outer while loop
|
||||||
|
else:
|
||||||
|
if (0, state) in arcs:
|
||||||
|
# An accepting state, pop it and try something else
|
||||||
|
self.pop()
|
||||||
|
if not self.stack:
|
||||||
|
# Done parsing, but another token is input
|
||||||
|
raise ParseError("too much input",
|
||||||
|
type, value, context)
|
||||||
|
else:
|
||||||
|
# No success finding a transition
|
||||||
|
raise ParseError("bad input", type, value, context)
|
||||||
|
|
||||||
|
def classify(self, type, value, context):
|
||||||
|
"""Turn a token into a label. (Internal)"""
|
||||||
|
if type == token.NAME:
|
||||||
|
# Keep a listing of all used names
|
||||||
|
self.used_names.add(value)
|
||||||
|
# Check for reserved words
|
||||||
|
ilabel = self.grammar.keywords.get(value)
|
||||||
|
if ilabel is not None:
|
||||||
|
return ilabel
|
||||||
|
ilabel = self.grammar.tokens.get(type)
|
||||||
|
if ilabel is None:
|
||||||
|
raise ParseError("bad token", type, value, context)
|
||||||
|
return ilabel
|
||||||
|
|
||||||
|
def shift(self, type, value, newstate, context):
|
||||||
|
"""Shift a token. (Internal)"""
|
||||||
|
dfa, state, node = self.stack[-1]
|
||||||
|
newnode = (type, value, context, None)
|
||||||
|
newnode = self.convert(self.grammar, newnode)
|
||||||
|
if newnode is not None:
|
||||||
|
node[-1].append(newnode)
|
||||||
|
self.stack[-1] = (dfa, newstate, node)
|
||||||
|
|
||||||
|
def push(self, type, newdfa, newstate, context):
|
||||||
|
"""Push a nonterminal. (Internal)"""
|
||||||
|
dfa, state, node = self.stack[-1]
|
||||||
|
newnode = (type, None, context, [])
|
||||||
|
self.stack[-1] = (dfa, newstate, node)
|
||||||
|
self.stack.append((newdfa, 0, newnode))
|
||||||
|
|
||||||
|
def pop(self):
|
||||||
|
"""Pop a nonterminal. (Internal)"""
|
||||||
|
popdfa, popstate, popnode = self.stack.pop()
|
||||||
|
newnode = self.convert(self.grammar, popnode)
|
||||||
|
if newnode is not None:
|
||||||
|
if self.stack:
|
||||||
|
dfa, state, node = self.stack[-1]
|
||||||
|
node[-1].append(newnode)
|
||||||
|
else:
|
||||||
|
self.rootnode = newnode
|
||||||
|
self.rootnode.used_names = self.used_names
|
||||||
29
blib2to3/pgen2/parse.pyi
Normal file
29
blib2to3/pgen2/parse.pyi
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
# Stubs for lib2to3.pgen2.parse (Python 3.6)
|
||||||
|
|
||||||
|
from typing import Any, Dict, List, Optional, Sequence, Set, Text, Tuple
|
||||||
|
|
||||||
|
from blib2to3.pgen2.grammar import Grammar, _DFAS
|
||||||
|
from blib2to3.pytree import _NL, _Convert, _RawNode
|
||||||
|
|
||||||
|
_Context = Sequence[Any]
|
||||||
|
|
||||||
|
class ParseError(Exception):
|
||||||
|
msg: Text
|
||||||
|
type: int
|
||||||
|
value: Optional[Text]
|
||||||
|
context: _Context
|
||||||
|
def __init__(self, msg: Text, type: int, value: Optional[Text], context: _Context) -> None: ...
|
||||||
|
|
||||||
|
class Parser:
|
||||||
|
grammar: Grammar
|
||||||
|
convert: _Convert
|
||||||
|
stack: List[Tuple[_DFAS, int, _RawNode]]
|
||||||
|
rootnode: Optional[_NL]
|
||||||
|
used_names: Set[Text]
|
||||||
|
def __init__(self, grammar: Grammar, convert: Optional[_Convert] = ...) -> None: ...
|
||||||
|
def setup(self, start: Optional[int] = ...) -> None: ...
|
||||||
|
def addtoken(self, type: int, value: Optional[Text], context: _Context) -> bool: ...
|
||||||
|
def classify(self, type: int, value: Optional[Text], context: _Context) -> int: ...
|
||||||
|
def shift(self, type: int, value: Optional[Text], newstate: int, context: _Context) -> None: ...
|
||||||
|
def push(self, type: int, newdfa: _DFAS, newstate: int, context: _Context) -> None: ...
|
||||||
|
def pop(self) -> None: ...
|
||||||
386
blib2to3/pgen2/pgen.py
Normal file
386
blib2to3/pgen2/pgen.py
Normal file
@ -0,0 +1,386 @@
|
|||||||
|
# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
|
||||||
|
# Licensed to PSF under a Contributor Agreement.
|
||||||
|
|
||||||
|
# Pgen imports
|
||||||
|
from . import grammar, token, tokenize
|
||||||
|
|
||||||
|
class PgenGrammar(grammar.Grammar):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class ParserGenerator(object):
|
||||||
|
|
||||||
|
def __init__(self, filename, stream=None):
|
||||||
|
close_stream = None
|
||||||
|
if stream is None:
|
||||||
|
stream = open(filename)
|
||||||
|
close_stream = stream.close
|
||||||
|
self.filename = filename
|
||||||
|
self.stream = stream
|
||||||
|
self.generator = tokenize.generate_tokens(stream.readline)
|
||||||
|
self.gettoken() # Initialize lookahead
|
||||||
|
self.dfas, self.startsymbol = self.parse()
|
||||||
|
if close_stream is not None:
|
||||||
|
close_stream()
|
||||||
|
self.first = {} # map from symbol name to set of tokens
|
||||||
|
self.addfirstsets()
|
||||||
|
|
||||||
|
def make_grammar(self):
|
||||||
|
c = PgenGrammar()
|
||||||
|
names = list(self.dfas.keys())
|
||||||
|
names.sort()
|
||||||
|
names.remove(self.startsymbol)
|
||||||
|
names.insert(0, self.startsymbol)
|
||||||
|
for name in names:
|
||||||
|
i = 256 + len(c.symbol2number)
|
||||||
|
c.symbol2number[name] = i
|
||||||
|
c.number2symbol[i] = name
|
||||||
|
for name in names:
|
||||||
|
dfa = self.dfas[name]
|
||||||
|
states = []
|
||||||
|
for state in dfa:
|
||||||
|
arcs = []
|
||||||
|
for label, next in sorted(state.arcs.items()):
|
||||||
|
arcs.append((self.make_label(c, label), dfa.index(next)))
|
||||||
|
if state.isfinal:
|
||||||
|
arcs.append((0, dfa.index(state)))
|
||||||
|
states.append(arcs)
|
||||||
|
c.states.append(states)
|
||||||
|
c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name))
|
||||||
|
c.start = c.symbol2number[self.startsymbol]
|
||||||
|
return c
|
||||||
|
|
||||||
|
def make_first(self, c, name):
|
||||||
|
rawfirst = self.first[name]
|
||||||
|
first = {}
|
||||||
|
for label in sorted(rawfirst):
|
||||||
|
ilabel = self.make_label(c, label)
|
||||||
|
##assert ilabel not in first # XXX failed on <> ... !=
|
||||||
|
first[ilabel] = 1
|
||||||
|
return first
|
||||||
|
|
||||||
|
def make_label(self, c, label):
|
||||||
|
# XXX Maybe this should be a method on a subclass of converter?
|
||||||
|
ilabel = len(c.labels)
|
||||||
|
if label[0].isalpha():
|
||||||
|
# Either a symbol name or a named token
|
||||||
|
if label in c.symbol2number:
|
||||||
|
# A symbol name (a non-terminal)
|
||||||
|
if label in c.symbol2label:
|
||||||
|
return c.symbol2label[label]
|
||||||
|
else:
|
||||||
|
c.labels.append((c.symbol2number[label], None))
|
||||||
|
c.symbol2label[label] = ilabel
|
||||||
|
return ilabel
|
||||||
|
else:
|
||||||
|
# A named token (NAME, NUMBER, STRING)
|
||||||
|
itoken = getattr(token, label, None)
|
||||||
|
assert isinstance(itoken, int), label
|
||||||
|
assert itoken in token.tok_name, label
|
||||||
|
if itoken in c.tokens:
|
||||||
|
return c.tokens[itoken]
|
||||||
|
else:
|
||||||
|
c.labels.append((itoken, None))
|
||||||
|
c.tokens[itoken] = ilabel
|
||||||
|
return ilabel
|
||||||
|
else:
|
||||||
|
# Either a keyword or an operator
|
||||||
|
assert label[0] in ('"', "'"), label
|
||||||
|
value = eval(label)
|
||||||
|
if value[0].isalpha():
|
||||||
|
# A keyword
|
||||||
|
if value in c.keywords:
|
||||||
|
return c.keywords[value]
|
||||||
|
else:
|
||||||
|
c.labels.append((token.NAME, value))
|
||||||
|
c.keywords[value] = ilabel
|
||||||
|
return ilabel
|
||||||
|
else:
|
||||||
|
# An operator (any non-numeric token)
|
||||||
|
itoken = grammar.opmap[value] # Fails if unknown token
|
||||||
|
if itoken in c.tokens:
|
||||||
|
return c.tokens[itoken]
|
||||||
|
else:
|
||||||
|
c.labels.append((itoken, None))
|
||||||
|
c.tokens[itoken] = ilabel
|
||||||
|
return ilabel
|
||||||
|
|
||||||
|
def addfirstsets(self):
|
||||||
|
names = list(self.dfas.keys())
|
||||||
|
names.sort()
|
||||||
|
for name in names:
|
||||||
|
if name not in self.first:
|
||||||
|
self.calcfirst(name)
|
||||||
|
#print name, self.first[name].keys()
|
||||||
|
|
||||||
|
def calcfirst(self, name):
|
||||||
|
dfa = self.dfas[name]
|
||||||
|
self.first[name] = None # dummy to detect left recursion
|
||||||
|
state = dfa[0]
|
||||||
|
totalset = {}
|
||||||
|
overlapcheck = {}
|
||||||
|
for label, next in state.arcs.items():
|
||||||
|
if label in self.dfas:
|
||||||
|
if label in self.first:
|
||||||
|
fset = self.first[label]
|
||||||
|
if fset is None:
|
||||||
|
raise ValueError("recursion for rule %r" % name)
|
||||||
|
else:
|
||||||
|
self.calcfirst(label)
|
||||||
|
fset = self.first[label]
|
||||||
|
totalset.update(fset)
|
||||||
|
overlapcheck[label] = fset
|
||||||
|
else:
|
||||||
|
totalset[label] = 1
|
||||||
|
overlapcheck[label] = {label: 1}
|
||||||
|
inverse = {}
|
||||||
|
for label, itsfirst in overlapcheck.items():
|
||||||
|
for symbol in itsfirst:
|
||||||
|
if symbol in inverse:
|
||||||
|
raise ValueError("rule %s is ambiguous; %s is in the"
|
||||||
|
" first sets of %s as well as %s" %
|
||||||
|
(name, symbol, label, inverse[symbol]))
|
||||||
|
inverse[symbol] = label
|
||||||
|
self.first[name] = totalset
|
||||||
|
|
||||||
|
def parse(self):
|
||||||
|
dfas = {}
|
||||||
|
startsymbol = None
|
||||||
|
# MSTART: (NEWLINE | RULE)* ENDMARKER
|
||||||
|
while self.type != token.ENDMARKER:
|
||||||
|
while self.type == token.NEWLINE:
|
||||||
|
self.gettoken()
|
||||||
|
# RULE: NAME ':' RHS NEWLINE
|
||||||
|
name = self.expect(token.NAME)
|
||||||
|
self.expect(token.OP, ":")
|
||||||
|
a, z = self.parse_rhs()
|
||||||
|
self.expect(token.NEWLINE)
|
||||||
|
#self.dump_nfa(name, a, z)
|
||||||
|
dfa = self.make_dfa(a, z)
|
||||||
|
#self.dump_dfa(name, dfa)
|
||||||
|
oldlen = len(dfa)
|
||||||
|
self.simplify_dfa(dfa)
|
||||||
|
newlen = len(dfa)
|
||||||
|
dfas[name] = dfa
|
||||||
|
#print name, oldlen, newlen
|
||||||
|
if startsymbol is None:
|
||||||
|
startsymbol = name
|
||||||
|
return dfas, startsymbol
|
||||||
|
|
||||||
|
def make_dfa(self, start, finish):
|
||||||
|
# To turn an NFA into a DFA, we define the states of the DFA
|
||||||
|
# to correspond to *sets* of states of the NFA. Then do some
|
||||||
|
# state reduction. Let's represent sets as dicts with 1 for
|
||||||
|
# values.
|
||||||
|
assert isinstance(start, NFAState)
|
||||||
|
assert isinstance(finish, NFAState)
|
||||||
|
def closure(state):
|
||||||
|
base = {}
|
||||||
|
addclosure(state, base)
|
||||||
|
return base
|
||||||
|
def addclosure(state, base):
|
||||||
|
assert isinstance(state, NFAState)
|
||||||
|
if state in base:
|
||||||
|
return
|
||||||
|
base[state] = 1
|
||||||
|
for label, next in state.arcs:
|
||||||
|
if label is None:
|
||||||
|
addclosure(next, base)
|
||||||
|
states = [DFAState(closure(start), finish)]
|
||||||
|
for state in states: # NB states grows while we're iterating
|
||||||
|
arcs = {}
|
||||||
|
for nfastate in state.nfaset:
|
||||||
|
for label, next in nfastate.arcs:
|
||||||
|
if label is not None:
|
||||||
|
addclosure(next, arcs.setdefault(label, {}))
|
||||||
|
for label, nfaset in sorted(arcs.items()):
|
||||||
|
for st in states:
|
||||||
|
if st.nfaset == nfaset:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
st = DFAState(nfaset, finish)
|
||||||
|
states.append(st)
|
||||||
|
state.addarc(st, label)
|
||||||
|
return states # List of DFAState instances; first one is start
|
||||||
|
|
||||||
|
def dump_nfa(self, name, start, finish):
|
||||||
|
print("Dump of NFA for", name)
|
||||||
|
todo = [start]
|
||||||
|
for i, state in enumerate(todo):
|
||||||
|
print(" State", i, state is finish and "(final)" or "")
|
||||||
|
for label, next in state.arcs:
|
||||||
|
if next in todo:
|
||||||
|
j = todo.index(next)
|
||||||
|
else:
|
||||||
|
j = len(todo)
|
||||||
|
todo.append(next)
|
||||||
|
if label is None:
|
||||||
|
print(" -> %d" % j)
|
||||||
|
else:
|
||||||
|
print(" %s -> %d" % (label, j))
|
||||||
|
|
||||||
|
def dump_dfa(self, name, dfa):
|
||||||
|
print("Dump of DFA for", name)
|
||||||
|
for i, state in enumerate(dfa):
|
||||||
|
print(" State", i, state.isfinal and "(final)" or "")
|
||||||
|
for label, next in sorted(state.arcs.items()):
|
||||||
|
print(" %s -> %d" % (label, dfa.index(next)))
|
||||||
|
|
||||||
|
def simplify_dfa(self, dfa):
|
||||||
|
# This is not theoretically optimal, but works well enough.
|
||||||
|
# Algorithm: repeatedly look for two states that have the same
|
||||||
|
# set of arcs (same labels pointing to the same nodes) and
|
||||||
|
# unify them, until things stop changing.
|
||||||
|
|
||||||
|
# dfa is a list of DFAState instances
|
||||||
|
changes = True
|
||||||
|
while changes:
|
||||||
|
changes = False
|
||||||
|
for i, state_i in enumerate(dfa):
|
||||||
|
for j in range(i+1, len(dfa)):
|
||||||
|
state_j = dfa[j]
|
||||||
|
if state_i == state_j:
|
||||||
|
#print " unify", i, j
|
||||||
|
del dfa[j]
|
||||||
|
for state in dfa:
|
||||||
|
state.unifystate(state_j, state_i)
|
||||||
|
changes = True
|
||||||
|
break
|
||||||
|
|
||||||
|
def parse_rhs(self):
|
||||||
|
# RHS: ALT ('|' ALT)*
|
||||||
|
a, z = self.parse_alt()
|
||||||
|
if self.value != "|":
|
||||||
|
return a, z
|
||||||
|
else:
|
||||||
|
aa = NFAState()
|
||||||
|
zz = NFAState()
|
||||||
|
aa.addarc(a)
|
||||||
|
z.addarc(zz)
|
||||||
|
while self.value == "|":
|
||||||
|
self.gettoken()
|
||||||
|
a, z = self.parse_alt()
|
||||||
|
aa.addarc(a)
|
||||||
|
z.addarc(zz)
|
||||||
|
return aa, zz
|
||||||
|
|
||||||
|
def parse_alt(self):
|
||||||
|
# ALT: ITEM+
|
||||||
|
a, b = self.parse_item()
|
||||||
|
while (self.value in ("(", "[") or
|
||||||
|
self.type in (token.NAME, token.STRING)):
|
||||||
|
c, d = self.parse_item()
|
||||||
|
b.addarc(c)
|
||||||
|
b = d
|
||||||
|
return a, b
|
||||||
|
|
||||||
|
def parse_item(self):
|
||||||
|
# ITEM: '[' RHS ']' | ATOM ['+' | '*']
|
||||||
|
if self.value == "[":
|
||||||
|
self.gettoken()
|
||||||
|
a, z = self.parse_rhs()
|
||||||
|
self.expect(token.OP, "]")
|
||||||
|
a.addarc(z)
|
||||||
|
return a, z
|
||||||
|
else:
|
||||||
|
a, z = self.parse_atom()
|
||||||
|
value = self.value
|
||||||
|
if value not in ("+", "*"):
|
||||||
|
return a, z
|
||||||
|
self.gettoken()
|
||||||
|
z.addarc(a)
|
||||||
|
if value == "+":
|
||||||
|
return a, z
|
||||||
|
else:
|
||||||
|
return a, a
|
||||||
|
|
||||||
|
def parse_atom(self):
|
||||||
|
# ATOM: '(' RHS ')' | NAME | STRING
|
||||||
|
if self.value == "(":
|
||||||
|
self.gettoken()
|
||||||
|
a, z = self.parse_rhs()
|
||||||
|
self.expect(token.OP, ")")
|
||||||
|
return a, z
|
||||||
|
elif self.type in (token.NAME, token.STRING):
|
||||||
|
a = NFAState()
|
||||||
|
z = NFAState()
|
||||||
|
a.addarc(z, self.value)
|
||||||
|
self.gettoken()
|
||||||
|
return a, z
|
||||||
|
else:
|
||||||
|
self.raise_error("expected (...) or NAME or STRING, got %s/%s",
|
||||||
|
self.type, self.value)
|
||||||
|
|
||||||
|
def expect(self, type, value=None):
|
||||||
|
if self.type != type or (value is not None and self.value != value):
|
||||||
|
self.raise_error("expected %s/%s, got %s/%s",
|
||||||
|
type, value, self.type, self.value)
|
||||||
|
value = self.value
|
||||||
|
self.gettoken()
|
||||||
|
return value
|
||||||
|
|
||||||
|
def gettoken(self):
|
||||||
|
tup = next(self.generator)
|
||||||
|
while tup[0] in (tokenize.COMMENT, tokenize.NL):
|
||||||
|
tup = next(self.generator)
|
||||||
|
self.type, self.value, self.begin, self.end, self.line = tup
|
||||||
|
#print token.tok_name[self.type], repr(self.value)
|
||||||
|
|
||||||
|
def raise_error(self, msg, *args):
|
||||||
|
if args:
|
||||||
|
try:
|
||||||
|
msg = msg % args
|
||||||
|
except:
|
||||||
|
msg = " ".join([msg] + list(map(str, args)))
|
||||||
|
raise SyntaxError(msg, (self.filename, self.end[0],
|
||||||
|
self.end[1], self.line))
|
||||||
|
|
||||||
|
class NFAState(object):
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.arcs = [] # list of (label, NFAState) pairs
|
||||||
|
|
||||||
|
def addarc(self, next, label=None):
|
||||||
|
assert label is None or isinstance(label, str)
|
||||||
|
assert isinstance(next, NFAState)
|
||||||
|
self.arcs.append((label, next))
|
||||||
|
|
||||||
|
class DFAState(object):
|
||||||
|
|
||||||
|
def __init__(self, nfaset, final):
|
||||||
|
assert isinstance(nfaset, dict)
|
||||||
|
assert isinstance(next(iter(nfaset)), NFAState)
|
||||||
|
assert isinstance(final, NFAState)
|
||||||
|
self.nfaset = nfaset
|
||||||
|
self.isfinal = final in nfaset
|
||||||
|
self.arcs = {} # map from label to DFAState
|
||||||
|
|
||||||
|
def addarc(self, next, label):
|
||||||
|
assert isinstance(label, str)
|
||||||
|
assert label not in self.arcs
|
||||||
|
assert isinstance(next, DFAState)
|
||||||
|
self.arcs[label] = next
|
||||||
|
|
||||||
|
def unifystate(self, old, new):
|
||||||
|
for label, next in self.arcs.items():
|
||||||
|
if next is old:
|
||||||
|
self.arcs[label] = new
|
||||||
|
|
||||||
|
def __eq__(self, other):
|
||||||
|
# Equality test -- ignore the nfaset instance variable
|
||||||
|
assert isinstance(other, DFAState)
|
||||||
|
if self.isfinal != other.isfinal:
|
||||||
|
return False
|
||||||
|
# Can't just return self.arcs == other.arcs, because that
|
||||||
|
# would invoke this method recursively, with cycles...
|
||||||
|
if len(self.arcs) != len(other.arcs):
|
||||||
|
return False
|
||||||
|
for label, next in self.arcs.items():
|
||||||
|
if next is not other.arcs.get(label):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
__hash__ = None # For Py3 compatibility.
|
||||||
|
|
||||||
|
def generate_grammar(filename="Grammar.txt"):
|
||||||
|
p = ParserGenerator(filename)
|
||||||
|
return p.make_grammar()
|
||||||
49
blib2to3/pgen2/pgen.pyi
Normal file
49
blib2to3/pgen2/pgen.pyi
Normal file
@ -0,0 +1,49 @@
|
|||||||
|
# Stubs for lib2to3.pgen2.pgen (Python 3.6)
|
||||||
|
|
||||||
|
from typing import Any, Dict, IO, Iterable, Iterator, List, Optional, Text, Tuple
|
||||||
|
from mypy_extensions import NoReturn
|
||||||
|
|
||||||
|
from blib2to3.pgen2 import _Path, grammar
|
||||||
|
from blib2to3.pgen2.tokenize import _TokenInfo
|
||||||
|
|
||||||
|
class PgenGrammar(grammar.Grammar): ...
|
||||||
|
|
||||||
|
class ParserGenerator:
|
||||||
|
filename: _Path
|
||||||
|
stream: IO[Text]
|
||||||
|
generator: Iterator[_TokenInfo]
|
||||||
|
first: Dict[Text, Dict[Text, int]]
|
||||||
|
def __init__(self, filename: _Path, stream: Optional[IO[Text]] = ...) -> None: ...
|
||||||
|
def make_grammar(self) -> PgenGrammar: ...
|
||||||
|
def make_first(self, c: PgenGrammar, name: Text) -> Dict[int, int]: ...
|
||||||
|
def make_label(self, c: PgenGrammar, label: Text) -> int: ...
|
||||||
|
def addfirstsets(self) -> None: ...
|
||||||
|
def calcfirst(self, name: Text) -> None: ...
|
||||||
|
def parse(self) -> Tuple[Dict[Text, List[DFAState]], Text]: ...
|
||||||
|
def make_dfa(self, start: NFAState, finish: NFAState) -> List[DFAState]: ...
|
||||||
|
def dump_nfa(self, name: Text, start: NFAState, finish: NFAState) -> List[DFAState]: ...
|
||||||
|
def dump_dfa(self, name: Text, dfa: Iterable[DFAState]) -> None: ...
|
||||||
|
def simplify_dfa(self, dfa: List[DFAState]) -> None: ...
|
||||||
|
def parse_rhs(self) -> Tuple[NFAState, NFAState]: ...
|
||||||
|
def parse_alt(self) -> Tuple[NFAState, NFAState]: ...
|
||||||
|
def parse_item(self) -> Tuple[NFAState, NFAState]: ...
|
||||||
|
def parse_atom(self) -> Tuple[NFAState, NFAState]: ...
|
||||||
|
def expect(self, type: int, value: Optional[Any] = ...) -> Text: ...
|
||||||
|
def gettoken(self) -> None: ...
|
||||||
|
def raise_error(self, msg: str, *args: Any) -> NoReturn: ...
|
||||||
|
|
||||||
|
class NFAState:
|
||||||
|
arcs: List[Tuple[Optional[Text], NFAState]]
|
||||||
|
def __init__(self) -> None: ...
|
||||||
|
def addarc(self, next: NFAState, label: Optional[Text] = ...) -> None: ...
|
||||||
|
|
||||||
|
class DFAState:
|
||||||
|
nfaset: Dict[NFAState, Any]
|
||||||
|
isfinal: bool
|
||||||
|
arcs: Dict[Text, DFAState]
|
||||||
|
def __init__(self, nfaset: Dict[NFAState, Any], final: NFAState) -> None: ...
|
||||||
|
def addarc(self, next: DFAState, label: Text) -> None: ...
|
||||||
|
def unifystate(self, old: DFAState, new: DFAState) -> None: ...
|
||||||
|
def __eq__(self, other: Any) -> bool: ...
|
||||||
|
|
||||||
|
def generate_grammar(filename: _Path = ...) -> PgenGrammar: ...
|
||||||
83
blib2to3/pgen2/token.py
Executable file
83
blib2to3/pgen2/token.py
Executable file
@ -0,0 +1,83 @@
|
|||||||
|
#! /usr/bin/env python3
|
||||||
|
|
||||||
|
"""Token constants (from "token.h")."""
|
||||||
|
|
||||||
|
# Taken from Python (r53757) and modified to include some tokens
|
||||||
|
# originally monkeypatched in by pgen2.tokenize
|
||||||
|
|
||||||
|
#--start constants--
|
||||||
|
ENDMARKER = 0
|
||||||
|
NAME = 1
|
||||||
|
NUMBER = 2
|
||||||
|
STRING = 3
|
||||||
|
NEWLINE = 4
|
||||||
|
INDENT = 5
|
||||||
|
DEDENT = 6
|
||||||
|
LPAR = 7
|
||||||
|
RPAR = 8
|
||||||
|
LSQB = 9
|
||||||
|
RSQB = 10
|
||||||
|
COLON = 11
|
||||||
|
COMMA = 12
|
||||||
|
SEMI = 13
|
||||||
|
PLUS = 14
|
||||||
|
MINUS = 15
|
||||||
|
STAR = 16
|
||||||
|
SLASH = 17
|
||||||
|
VBAR = 18
|
||||||
|
AMPER = 19
|
||||||
|
LESS = 20
|
||||||
|
GREATER = 21
|
||||||
|
EQUAL = 22
|
||||||
|
DOT = 23
|
||||||
|
PERCENT = 24
|
||||||
|
BACKQUOTE = 25
|
||||||
|
LBRACE = 26
|
||||||
|
RBRACE = 27
|
||||||
|
EQEQUAL = 28
|
||||||
|
NOTEQUAL = 29
|
||||||
|
LESSEQUAL = 30
|
||||||
|
GREATEREQUAL = 31
|
||||||
|
TILDE = 32
|
||||||
|
CIRCUMFLEX = 33
|
||||||
|
LEFTSHIFT = 34
|
||||||
|
RIGHTSHIFT = 35
|
||||||
|
DOUBLESTAR = 36
|
||||||
|
PLUSEQUAL = 37
|
||||||
|
MINEQUAL = 38
|
||||||
|
STAREQUAL = 39
|
||||||
|
SLASHEQUAL = 40
|
||||||
|
PERCENTEQUAL = 41
|
||||||
|
AMPEREQUAL = 42
|
||||||
|
VBAREQUAL = 43
|
||||||
|
CIRCUMFLEXEQUAL = 44
|
||||||
|
LEFTSHIFTEQUAL = 45
|
||||||
|
RIGHTSHIFTEQUAL = 46
|
||||||
|
DOUBLESTAREQUAL = 47
|
||||||
|
DOUBLESLASH = 48
|
||||||
|
DOUBLESLASHEQUAL = 49
|
||||||
|
AT = 50
|
||||||
|
ATEQUAL = 51
|
||||||
|
OP = 52
|
||||||
|
COMMENT = 53
|
||||||
|
NL = 54
|
||||||
|
RARROW = 55
|
||||||
|
ERRORTOKEN = 56
|
||||||
|
N_TOKENS = 57
|
||||||
|
NT_OFFSET = 256
|
||||||
|
#--end constants--
|
||||||
|
|
||||||
|
tok_name = {}
|
||||||
|
for _name, _value in list(globals().items()):
|
||||||
|
if type(_value) is type(0):
|
||||||
|
tok_name[_value] = _name
|
||||||
|
|
||||||
|
|
||||||
|
def ISTERMINAL(x):
|
||||||
|
return x < NT_OFFSET
|
||||||
|
|
||||||
|
def ISNONTERMINAL(x):
|
||||||
|
return x >= NT_OFFSET
|
||||||
|
|
||||||
|
def ISEOF(x):
|
||||||
|
return x == ENDMARKER
|
||||||
73
blib2to3/pgen2/token.pyi
Normal file
73
blib2to3/pgen2/token.pyi
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
# Stubs for lib2to3.pgen2.token (Python 3.6)
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from typing import Dict, Text
|
||||||
|
|
||||||
|
ENDMARKER: int
|
||||||
|
NAME: int
|
||||||
|
NUMBER: int
|
||||||
|
STRING: int
|
||||||
|
NEWLINE: int
|
||||||
|
INDENT: int
|
||||||
|
DEDENT: int
|
||||||
|
LPAR: int
|
||||||
|
RPAR: int
|
||||||
|
LSQB: int
|
||||||
|
RSQB: int
|
||||||
|
COLON: int
|
||||||
|
COMMA: int
|
||||||
|
SEMI: int
|
||||||
|
PLUS: int
|
||||||
|
MINUS: int
|
||||||
|
STAR: int
|
||||||
|
SLASH: int
|
||||||
|
VBAR: int
|
||||||
|
AMPER: int
|
||||||
|
LESS: int
|
||||||
|
GREATER: int
|
||||||
|
EQUAL: int
|
||||||
|
DOT: int
|
||||||
|
PERCENT: int
|
||||||
|
BACKQUOTE: int
|
||||||
|
LBRACE: int
|
||||||
|
RBRACE: int
|
||||||
|
EQEQUAL: int
|
||||||
|
NOTEQUAL: int
|
||||||
|
LESSEQUAL: int
|
||||||
|
GREATEREQUAL: int
|
||||||
|
TILDE: int
|
||||||
|
CIRCUMFLEX: int
|
||||||
|
LEFTSHIFT: int
|
||||||
|
RIGHTSHIFT: int
|
||||||
|
DOUBLESTAR: int
|
||||||
|
PLUSEQUAL: int
|
||||||
|
MINEQUAL: int
|
||||||
|
STAREQUAL: int
|
||||||
|
SLASHEQUAL: int
|
||||||
|
PERCENTEQUAL: int
|
||||||
|
AMPEREQUAL: int
|
||||||
|
VBAREQUAL: int
|
||||||
|
CIRCUMFLEXEQUAL: int
|
||||||
|
LEFTSHIFTEQUAL: int
|
||||||
|
RIGHTSHIFTEQUAL: int
|
||||||
|
DOUBLESTAREQUAL: int
|
||||||
|
DOUBLESLASH: int
|
||||||
|
DOUBLESLASHEQUAL: int
|
||||||
|
OP: int
|
||||||
|
COMMENT: int
|
||||||
|
NL: int
|
||||||
|
if sys.version_info >= (3,):
|
||||||
|
RARROW: int
|
||||||
|
if sys.version_info >= (3, 5):
|
||||||
|
AT: int
|
||||||
|
ATEQUAL: int
|
||||||
|
AWAIT: int
|
||||||
|
ASYNC: int
|
||||||
|
ERRORTOKEN: int
|
||||||
|
N_TOKENS: int
|
||||||
|
NT_OFFSET: int
|
||||||
|
tok_name: Dict[int, Text]
|
||||||
|
|
||||||
|
def ISTERMINAL(x: int) -> bool: ...
|
||||||
|
def ISNONTERMINAL(x: int) -> bool: ...
|
||||||
|
def ISEOF(x: int) -> bool: ...
|
||||||
518
blib2to3/pgen2/tokenize.py
Normal file
518
blib2to3/pgen2/tokenize.py
Normal file
@ -0,0 +1,518 @@
|
|||||||
|
# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation.
|
||||||
|
# All rights reserved.
|
||||||
|
|
||||||
|
"""Tokenization help for Python programs.
|
||||||
|
|
||||||
|
generate_tokens(readline) is a generator that breaks a stream of
|
||||||
|
text into Python tokens. It accepts a readline-like method which is called
|
||||||
|
repeatedly to get the next line of input (or "" for EOF). It generates
|
||||||
|
5-tuples with these members:
|
||||||
|
|
||||||
|
the token type (see token.py)
|
||||||
|
the token (a string)
|
||||||
|
the starting (row, column) indices of the token (a 2-tuple of ints)
|
||||||
|
the ending (row, column) indices of the token (a 2-tuple of ints)
|
||||||
|
the original line (string)
|
||||||
|
|
||||||
|
It is designed to match the working of the Python tokenizer exactly, except
|
||||||
|
that it produces COMMENT tokens for comments and gives type OP for all
|
||||||
|
operators
|
||||||
|
|
||||||
|
Older entry points
|
||||||
|
tokenize_loop(readline, tokeneater)
|
||||||
|
tokenize(readline, tokeneater=printtoken)
|
||||||
|
are the same, except instead of generating tokens, tokeneater is a callback
|
||||||
|
function to which the 5 fields described above are passed as 5 arguments,
|
||||||
|
each time a new token is found."""
|
||||||
|
|
||||||
|
__author__ = 'Ka-Ping Yee <ping@lfw.org>'
|
||||||
|
__credits__ = \
|
||||||
|
'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro'
|
||||||
|
|
||||||
|
import string, re
|
||||||
|
from codecs import BOM_UTF8, lookup
|
||||||
|
from lib2to3.pgen2.token import *
|
||||||
|
|
||||||
|
from . import token
|
||||||
|
__all__ = [x for x in dir(token) if x[0] != '_'] + ["tokenize",
|
||||||
|
"generate_tokens", "untokenize"]
|
||||||
|
del token
|
||||||
|
|
||||||
|
try:
|
||||||
|
bytes
|
||||||
|
except NameError:
|
||||||
|
# Support bytes type in Python <= 2.5, so 2to3 turns itself into
|
||||||
|
# valid Python 3 code.
|
||||||
|
bytes = str
|
||||||
|
|
||||||
|
def group(*choices): return '(' + '|'.join(choices) + ')'
|
||||||
|
def any(*choices): return group(*choices) + '*'
|
||||||
|
def maybe(*choices): return group(*choices) + '?'
|
||||||
|
|
||||||
|
Whitespace = r'[ \f\t]*'
|
||||||
|
Comment = r'#[^\r\n]*'
|
||||||
|
Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
|
||||||
|
Name = r'[a-zA-Z_]\w*'
|
||||||
|
|
||||||
|
Binnumber = r'0[bB]_?[01]+(?:_[01]+)*'
|
||||||
|
Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?'
|
||||||
|
Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?'
|
||||||
|
Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?')
|
||||||
|
Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber)
|
||||||
|
Exponent = r'[eE][-+]?\d+(?:_\d+)*'
|
||||||
|
Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent)
|
||||||
|
Expfloat = r'\d+(?:_\d+)*' + Exponent
|
||||||
|
Floatnumber = group(Pointfloat, Expfloat)
|
||||||
|
Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]')
|
||||||
|
Number = group(Imagnumber, Floatnumber, Intnumber)
|
||||||
|
|
||||||
|
# Tail end of ' string.
|
||||||
|
Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
|
||||||
|
# Tail end of " string.
|
||||||
|
Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
|
||||||
|
# Tail end of ''' string.
|
||||||
|
Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
|
||||||
|
# Tail end of """ string.
|
||||||
|
Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
|
||||||
|
_litprefix = r"(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?"
|
||||||
|
Triple = group(_litprefix + "'''", _litprefix + '"""')
|
||||||
|
# Single-line ' or " string.
|
||||||
|
String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
|
||||||
|
_litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
|
||||||
|
|
||||||
|
# Because of leftmost-then-longest match semantics, be sure to put the
|
||||||
|
# longest operators first (e.g., if = came before ==, == would get
|
||||||
|
# recognized as two instances of =).
|
||||||
|
Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
|
||||||
|
r"//=?", r"->",
|
||||||
|
r"[+\-*/%&@|^=<>]=?",
|
||||||
|
r"~")
|
||||||
|
|
||||||
|
Bracket = '[][(){}]'
|
||||||
|
Special = group(r'\r?\n', r'[:;.,`@]')
|
||||||
|
Funny = group(Operator, Bracket, Special)
|
||||||
|
|
||||||
|
PlainToken = group(Number, Funny, String, Name)
|
||||||
|
Token = Ignore + PlainToken
|
||||||
|
|
||||||
|
# First (or only) line of ' or " string.
|
||||||
|
ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
|
||||||
|
group("'", r'\\\r?\n'),
|
||||||
|
_litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
|
||||||
|
group('"', r'\\\r?\n'))
|
||||||
|
PseudoExtras = group(r'\\\r?\n', Comment, Triple)
|
||||||
|
PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
|
||||||
|
|
||||||
|
tokenprog, pseudoprog, single3prog, double3prog = list(map(
|
||||||
|
re.compile, (Token, PseudoToken, Single3, Double3)))
|
||||||
|
endprogs = {"'": re.compile(Single), '"': re.compile(Double),
|
||||||
|
"'''": single3prog, '"""': double3prog,
|
||||||
|
"r'''": single3prog, 'r"""': double3prog,
|
||||||
|
"u'''": single3prog, 'u"""': double3prog,
|
||||||
|
"b'''": single3prog, 'b"""': double3prog,
|
||||||
|
"f'''": single3prog, 'f"""': double3prog,
|
||||||
|
"ur'''": single3prog, 'ur"""': double3prog,
|
||||||
|
"br'''": single3prog, 'br"""': double3prog,
|
||||||
|
"rb'''": single3prog, 'rb"""': double3prog,
|
||||||
|
"R'''": single3prog, 'R"""': double3prog,
|
||||||
|
"U'''": single3prog, 'U"""': double3prog,
|
||||||
|
"B'''": single3prog, 'B"""': double3prog,
|
||||||
|
"F'''": single3prog, 'F"""': double3prog,
|
||||||
|
"uR'''": single3prog, 'uR"""': double3prog,
|
||||||
|
"Ur'''": single3prog, 'Ur"""': double3prog,
|
||||||
|
"UR'''": single3prog, 'UR"""': double3prog,
|
||||||
|
"bR'''": single3prog, 'bR"""': double3prog,
|
||||||
|
"Br'''": single3prog, 'Br"""': double3prog,
|
||||||
|
"BR'''": single3prog, 'BR"""': double3prog,
|
||||||
|
"rB'''": single3prog, 'rB"""': double3prog,
|
||||||
|
"Rb'''": single3prog, 'Rb"""': double3prog,
|
||||||
|
"RB'''": single3prog, 'RB"""': double3prog,
|
||||||
|
'r': None, 'R': None,
|
||||||
|
'u': None, 'U': None,
|
||||||
|
'f': None, 'F': None,
|
||||||
|
'b': None, 'B': None}
|
||||||
|
|
||||||
|
triple_quoted = {}
|
||||||
|
for t in ("'''", '"""',
|
||||||
|
"r'''", 'r"""', "R'''", 'R"""',
|
||||||
|
"u'''", 'u"""', "U'''", 'U"""',
|
||||||
|
"b'''", 'b"""', "B'''", 'B"""',
|
||||||
|
"f'''", 'f"""', "F'''", 'F"""',
|
||||||
|
"ur'''", 'ur"""', "Ur'''", 'Ur"""',
|
||||||
|
"uR'''", 'uR"""', "UR'''", 'UR"""',
|
||||||
|
"br'''", 'br"""', "Br'''", 'Br"""',
|
||||||
|
"bR'''", 'bR"""', "BR'''", 'BR"""',
|
||||||
|
"rb'''", 'rb"""', "Rb'''", 'Rb"""',
|
||||||
|
"rB'''", 'rB"""', "RB'''", 'RB"""',):
|
||||||
|
triple_quoted[t] = t
|
||||||
|
single_quoted = {}
|
||||||
|
for t in ("'", '"',
|
||||||
|
"r'", 'r"', "R'", 'R"',
|
||||||
|
"u'", 'u"', "U'", 'U"',
|
||||||
|
"b'", 'b"', "B'", 'B"',
|
||||||
|
"f'", 'f"', "F'", 'F"',
|
||||||
|
"ur'", 'ur"', "Ur'", 'Ur"',
|
||||||
|
"uR'", 'uR"', "UR'", 'UR"',
|
||||||
|
"br'", 'br"', "Br'", 'Br"',
|
||||||
|
"bR'", 'bR"', "BR'", 'BR"',
|
||||||
|
"rb'", 'rb"', "Rb'", 'Rb"',
|
||||||
|
"rB'", 'rB"', "RB'", 'RB"',):
|
||||||
|
single_quoted[t] = t
|
||||||
|
|
||||||
|
tabsize = 8
|
||||||
|
|
||||||
|
class TokenError(Exception): pass
|
||||||
|
|
||||||
|
class StopTokenizing(Exception): pass
|
||||||
|
|
||||||
|
def printtoken(type, token, xxx_todo_changeme, xxx_todo_changeme1, line): # for testing
|
||||||
|
(srow, scol) = xxx_todo_changeme
|
||||||
|
(erow, ecol) = xxx_todo_changeme1
|
||||||
|
print("%d,%d-%d,%d:\t%s\t%s" % \
|
||||||
|
(srow, scol, erow, ecol, tok_name[type], repr(token)))
|
||||||
|
|
||||||
|
def tokenize(readline, tokeneater=printtoken):
|
||||||
|
"""
|
||||||
|
The tokenize() function accepts two parameters: one representing the
|
||||||
|
input stream, and one providing an output mechanism for tokenize().
|
||||||
|
|
||||||
|
The first parameter, readline, must be a callable object which provides
|
||||||
|
the same interface as the readline() method of built-in file objects.
|
||||||
|
Each call to the function should return one line of input as a string.
|
||||||
|
|
||||||
|
The second parameter, tokeneater, must also be a callable object. It is
|
||||||
|
called once for each token, with five arguments, corresponding to the
|
||||||
|
tuples generated by generate_tokens().
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
tokenize_loop(readline, tokeneater)
|
||||||
|
except StopTokenizing:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# backwards compatible interface
|
||||||
|
def tokenize_loop(readline, tokeneater):
|
||||||
|
for token_info in generate_tokens(readline):
|
||||||
|
tokeneater(*token_info)
|
||||||
|
|
||||||
|
class Untokenizer:
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.tokens = []
|
||||||
|
self.prev_row = 1
|
||||||
|
self.prev_col = 0
|
||||||
|
|
||||||
|
def add_whitespace(self, start):
|
||||||
|
row, col = start
|
||||||
|
assert row <= self.prev_row
|
||||||
|
col_offset = col - self.prev_col
|
||||||
|
if col_offset:
|
||||||
|
self.tokens.append(" " * col_offset)
|
||||||
|
|
||||||
|
def untokenize(self, iterable):
|
||||||
|
for t in iterable:
|
||||||
|
if len(t) == 2:
|
||||||
|
self.compat(t, iterable)
|
||||||
|
break
|
||||||
|
tok_type, token, start, end, line = t
|
||||||
|
self.add_whitespace(start)
|
||||||
|
self.tokens.append(token)
|
||||||
|
self.prev_row, self.prev_col = end
|
||||||
|
if tok_type in (NEWLINE, NL):
|
||||||
|
self.prev_row += 1
|
||||||
|
self.prev_col = 0
|
||||||
|
return "".join(self.tokens)
|
||||||
|
|
||||||
|
def compat(self, token, iterable):
|
||||||
|
startline = False
|
||||||
|
indents = []
|
||||||
|
toks_append = self.tokens.append
|
||||||
|
toknum, tokval = token
|
||||||
|
if toknum in (NAME, NUMBER):
|
||||||
|
tokval += ' '
|
||||||
|
if toknum in (NEWLINE, NL):
|
||||||
|
startline = True
|
||||||
|
for tok in iterable:
|
||||||
|
toknum, tokval = tok[:2]
|
||||||
|
|
||||||
|
if toknum in (NAME, NUMBER):
|
||||||
|
tokval += ' '
|
||||||
|
|
||||||
|
if toknum == INDENT:
|
||||||
|
indents.append(tokval)
|
||||||
|
continue
|
||||||
|
elif toknum == DEDENT:
|
||||||
|
indents.pop()
|
||||||
|
continue
|
||||||
|
elif toknum in (NEWLINE, NL):
|
||||||
|
startline = True
|
||||||
|
elif startline and indents:
|
||||||
|
toks_append(indents[-1])
|
||||||
|
startline = False
|
||||||
|
toks_append(tokval)
|
||||||
|
|
||||||
|
cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII)
|
||||||
|
blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII)
|
||||||
|
|
||||||
|
def _get_normal_name(orig_enc):
|
||||||
|
"""Imitates get_normal_name in tokenizer.c."""
|
||||||
|
# Only care about the first 12 characters.
|
||||||
|
enc = orig_enc[:12].lower().replace("_", "-")
|
||||||
|
if enc == "utf-8" or enc.startswith("utf-8-"):
|
||||||
|
return "utf-8"
|
||||||
|
if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
|
||||||
|
enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
|
||||||
|
return "iso-8859-1"
|
||||||
|
return orig_enc
|
||||||
|
|
||||||
|
def detect_encoding(readline):
|
||||||
|
"""
|
||||||
|
The detect_encoding() function is used to detect the encoding that should
|
||||||
|
be used to decode a Python source file. It requires one argument, readline,
|
||||||
|
in the same way as the tokenize() generator.
|
||||||
|
|
||||||
|
It will call readline a maximum of twice, and return the encoding used
|
||||||
|
(as a string) and a list of any lines (left as bytes) it has read
|
||||||
|
in.
|
||||||
|
|
||||||
|
It detects the encoding from the presence of a utf-8 bom or an encoding
|
||||||
|
cookie as specified in pep-0263. If both a bom and a cookie are present, but
|
||||||
|
disagree, a SyntaxError will be raised. If the encoding cookie is an invalid
|
||||||
|
charset, raise a SyntaxError. Note that if a utf-8 bom is found,
|
||||||
|
'utf-8-sig' is returned.
|
||||||
|
|
||||||
|
If no encoding is specified, then the default of 'utf-8' will be returned.
|
||||||
|
"""
|
||||||
|
bom_found = False
|
||||||
|
encoding = None
|
||||||
|
default = 'utf-8'
|
||||||
|
def read_or_stop():
|
||||||
|
try:
|
||||||
|
return readline()
|
||||||
|
except StopIteration:
|
||||||
|
return bytes()
|
||||||
|
|
||||||
|
def find_cookie(line):
|
||||||
|
try:
|
||||||
|
line_string = line.decode('ascii')
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
return None
|
||||||
|
match = cookie_re.match(line_string)
|
||||||
|
if not match:
|
||||||
|
return None
|
||||||
|
encoding = _get_normal_name(match.group(1))
|
||||||
|
try:
|
||||||
|
codec = lookup(encoding)
|
||||||
|
except LookupError:
|
||||||
|
# This behaviour mimics the Python interpreter
|
||||||
|
raise SyntaxError("unknown encoding: " + encoding)
|
||||||
|
|
||||||
|
if bom_found:
|
||||||
|
if codec.name != 'utf-8':
|
||||||
|
# This behaviour mimics the Python interpreter
|
||||||
|
raise SyntaxError('encoding problem: utf-8')
|
||||||
|
encoding += '-sig'
|
||||||
|
return encoding
|
||||||
|
|
||||||
|
first = read_or_stop()
|
||||||
|
if first.startswith(BOM_UTF8):
|
||||||
|
bom_found = True
|
||||||
|
first = first[3:]
|
||||||
|
default = 'utf-8-sig'
|
||||||
|
if not first:
|
||||||
|
return default, []
|
||||||
|
|
||||||
|
encoding = find_cookie(first)
|
||||||
|
if encoding:
|
||||||
|
return encoding, [first]
|
||||||
|
if not blank_re.match(first):
|
||||||
|
return default, [first]
|
||||||
|
|
||||||
|
second = read_or_stop()
|
||||||
|
if not second:
|
||||||
|
return default, [first]
|
||||||
|
|
||||||
|
encoding = find_cookie(second)
|
||||||
|
if encoding:
|
||||||
|
return encoding, [first, second]
|
||||||
|
|
||||||
|
return default, [first, second]
|
||||||
|
|
||||||
|
def untokenize(iterable):
|
||||||
|
"""Transform tokens back into Python source code.
|
||||||
|
|
||||||
|
Each element returned by the iterable must be a token sequence
|
||||||
|
with at least two elements, a token number and token value. If
|
||||||
|
only two tokens are passed, the resulting output is poor.
|
||||||
|
|
||||||
|
Round-trip invariant for full input:
|
||||||
|
Untokenized source will match input source exactly
|
||||||
|
|
||||||
|
Round-trip invariant for limited intput:
|
||||||
|
# Output text will tokenize the back to the input
|
||||||
|
t1 = [tok[:2] for tok in generate_tokens(f.readline)]
|
||||||
|
newcode = untokenize(t1)
|
||||||
|
readline = iter(newcode.splitlines(1)).next
|
||||||
|
t2 = [tok[:2] for tokin generate_tokens(readline)]
|
||||||
|
assert t1 == t2
|
||||||
|
"""
|
||||||
|
ut = Untokenizer()
|
||||||
|
return ut.untokenize(iterable)
|
||||||
|
|
||||||
|
def generate_tokens(readline):
|
||||||
|
"""
|
||||||
|
The generate_tokens() generator requires one argument, readline, which
|
||||||
|
must be a callable object which provides the same interface as the
|
||||||
|
readline() method of built-in file objects. Each call to the function
|
||||||
|
should return one line of input as a string. Alternately, readline
|
||||||
|
can be a callable function terminating with StopIteration:
|
||||||
|
readline = open(myfile).next # Example of alternate readline
|
||||||
|
|
||||||
|
The generator produces 5-tuples with these members: the token type; the
|
||||||
|
token string; a 2-tuple (srow, scol) of ints specifying the row and
|
||||||
|
column where the token begins in the source; a 2-tuple (erow, ecol) of
|
||||||
|
ints specifying the row and column where the token ends in the source;
|
||||||
|
and the line on which the token was found. The line passed is the
|
||||||
|
logical line; continuation lines are included.
|
||||||
|
"""
|
||||||
|
lnum = parenlev = continued = 0
|
||||||
|
namechars, numchars = string.ascii_letters + '_', '0123456789'
|
||||||
|
contstr, needcont = '', 0
|
||||||
|
contline = None
|
||||||
|
indents = [0]
|
||||||
|
|
||||||
|
while 1: # loop over lines in stream
|
||||||
|
try:
|
||||||
|
line = readline()
|
||||||
|
except StopIteration:
|
||||||
|
line = ''
|
||||||
|
lnum = lnum + 1
|
||||||
|
pos, max = 0, len(line)
|
||||||
|
|
||||||
|
if contstr: # continued string
|
||||||
|
if not line:
|
||||||
|
raise TokenError("EOF in multi-line string", strstart)
|
||||||
|
endmatch = endprog.match(line)
|
||||||
|
if endmatch:
|
||||||
|
pos = end = endmatch.end(0)
|
||||||
|
yield (STRING, contstr + line[:end],
|
||||||
|
strstart, (lnum, end), contline + line)
|
||||||
|
contstr, needcont = '', 0
|
||||||
|
contline = None
|
||||||
|
elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
|
||||||
|
yield (ERRORTOKEN, contstr + line,
|
||||||
|
strstart, (lnum, len(line)), contline)
|
||||||
|
contstr = ''
|
||||||
|
contline = None
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
contstr = contstr + line
|
||||||
|
contline = contline + line
|
||||||
|
continue
|
||||||
|
|
||||||
|
elif parenlev == 0 and not continued: # new statement
|
||||||
|
if not line: break
|
||||||
|
column = 0
|
||||||
|
while pos < max: # measure leading whitespace
|
||||||
|
if line[pos] == ' ': column = column + 1
|
||||||
|
elif line[pos] == '\t': column = (column//tabsize + 1)*tabsize
|
||||||
|
elif line[pos] == '\f': column = 0
|
||||||
|
else: break
|
||||||
|
pos = pos + 1
|
||||||
|
if pos == max: break
|
||||||
|
|
||||||
|
if line[pos] in '#\r\n': # skip comments or blank lines
|
||||||
|
if line[pos] == '#':
|
||||||
|
comment_token = line[pos:].rstrip('\r\n')
|
||||||
|
nl_pos = pos + len(comment_token)
|
||||||
|
yield (COMMENT, comment_token,
|
||||||
|
(lnum, pos), (lnum, pos + len(comment_token)), line)
|
||||||
|
yield (NL, line[nl_pos:],
|
||||||
|
(lnum, nl_pos), (lnum, len(line)), line)
|
||||||
|
else:
|
||||||
|
yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
|
||||||
|
(lnum, pos), (lnum, len(line)), line)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if column > indents[-1]: # count indents or dedents
|
||||||
|
indents.append(column)
|
||||||
|
yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
|
||||||
|
while column < indents[-1]:
|
||||||
|
if column not in indents:
|
||||||
|
raise IndentationError(
|
||||||
|
"unindent does not match any outer indentation level",
|
||||||
|
("<tokenize>", lnum, pos, line))
|
||||||
|
indents = indents[:-1]
|
||||||
|
|
||||||
|
yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
|
||||||
|
|
||||||
|
else: # continued statement
|
||||||
|
if not line:
|
||||||
|
raise TokenError("EOF in multi-line statement", (lnum, 0))
|
||||||
|
continued = 0
|
||||||
|
|
||||||
|
while pos < max:
|
||||||
|
pseudomatch = pseudoprog.match(line, pos)
|
||||||
|
if pseudomatch: # scan for tokens
|
||||||
|
start, end = pseudomatch.span(1)
|
||||||
|
spos, epos, pos = (lnum, start), (lnum, end), end
|
||||||
|
token, initial = line[start:end], line[start]
|
||||||
|
|
||||||
|
if initial in numchars or \
|
||||||
|
(initial == '.' and token != '.'): # ordinary number
|
||||||
|
yield (NUMBER, token, spos, epos, line)
|
||||||
|
elif initial in '\r\n':
|
||||||
|
newline = NEWLINE
|
||||||
|
if parenlev > 0:
|
||||||
|
newline = NL
|
||||||
|
yield (newline, token, spos, epos, line)
|
||||||
|
|
||||||
|
elif initial == '#':
|
||||||
|
assert not token.endswith("\n")
|
||||||
|
yield (COMMENT, token, spos, epos, line)
|
||||||
|
elif token in triple_quoted:
|
||||||
|
endprog = endprogs[token]
|
||||||
|
endmatch = endprog.match(line, pos)
|
||||||
|
if endmatch: # all on one line
|
||||||
|
pos = endmatch.end(0)
|
||||||
|
token = line[start:pos]
|
||||||
|
yield (STRING, token, spos, (lnum, pos), line)
|
||||||
|
else:
|
||||||
|
strstart = (lnum, start) # multiple lines
|
||||||
|
contstr = line[start:]
|
||||||
|
contline = line
|
||||||
|
break
|
||||||
|
elif initial in single_quoted or \
|
||||||
|
token[:2] in single_quoted or \
|
||||||
|
token[:3] in single_quoted:
|
||||||
|
if token[-1] == '\n': # continued string
|
||||||
|
strstart = (lnum, start)
|
||||||
|
endprog = (endprogs[initial] or endprogs[token[1]] or
|
||||||
|
endprogs[token[2]])
|
||||||
|
contstr, needcont = line[start:], 1
|
||||||
|
contline = line
|
||||||
|
break
|
||||||
|
else: # ordinary string
|
||||||
|
yield (STRING, token, spos, epos, line)
|
||||||
|
elif initial in namechars: # ordinary name
|
||||||
|
yield (NAME, token, spos, epos, line)
|
||||||
|
elif initial == '\\': # continued stmt
|
||||||
|
# This yield is new; needed for better idempotency:
|
||||||
|
yield (NL, token, spos, (lnum, pos), line)
|
||||||
|
continued = 1
|
||||||
|
else:
|
||||||
|
if initial in '([{': parenlev = parenlev + 1
|
||||||
|
elif initial in ')]}': parenlev = parenlev - 1
|
||||||
|
yield (OP, token, spos, epos, line)
|
||||||
|
else:
|
||||||
|
yield (ERRORTOKEN, line[pos],
|
||||||
|
(lnum, pos), (lnum, pos+1), line)
|
||||||
|
pos = pos + 1
|
||||||
|
|
||||||
|
for indent in indents[1:]: # pop remaining indent levels
|
||||||
|
yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
|
||||||
|
yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
|
||||||
|
|
||||||
|
if __name__ == '__main__': # testing
|
||||||
|
import sys
|
||||||
|
if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline)
|
||||||
|
else: tokenize(sys.stdin.readline)
|
||||||
30
blib2to3/pgen2/tokenize.pyi
Normal file
30
blib2to3/pgen2/tokenize.pyi
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
# Stubs for lib2to3.pgen2.tokenize (Python 3.6)
|
||||||
|
# NOTE: Only elements from __all__ are present.
|
||||||
|
|
||||||
|
from typing import Callable, Iterable, Iterator, List, Text, Tuple
|
||||||
|
from blib2to3.pgen2.token import * # noqa
|
||||||
|
|
||||||
|
|
||||||
|
_Coord = Tuple[int, int]
|
||||||
|
_TokenEater = Callable[[int, Text, _Coord, _Coord, Text], None]
|
||||||
|
_TokenInfo = Tuple[int, Text, _Coord, _Coord, Text]
|
||||||
|
|
||||||
|
|
||||||
|
class TokenError(Exception): ...
|
||||||
|
class StopTokenizing(Exception): ...
|
||||||
|
|
||||||
|
def tokenize(readline: Callable[[], Text], tokeneater: _TokenEater = ...) -> None: ...
|
||||||
|
|
||||||
|
class Untokenizer:
|
||||||
|
tokens: List[Text]
|
||||||
|
prev_row: int
|
||||||
|
prev_col: int
|
||||||
|
def __init__(self) -> None: ...
|
||||||
|
def add_whitespace(self, start: _Coord) -> None: ...
|
||||||
|
def untokenize(self, iterable: Iterable[_TokenInfo]) -> Text: ...
|
||||||
|
def compat(self, token: Tuple[int, Text], iterable: Iterable[_TokenInfo]) -> None: ...
|
||||||
|
|
||||||
|
def untokenize(iterable: Iterable[_TokenInfo]) -> Text: ...
|
||||||
|
def generate_tokens(
|
||||||
|
readline: Callable[[], Text]
|
||||||
|
) -> Iterator[_TokenInfo]: ...
|
||||||
40
blib2to3/pygram.py
Normal file
40
blib2to3/pygram.py
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
# Copyright 2006 Google, Inc. All Rights Reserved.
|
||||||
|
# Licensed to PSF under a Contributor Agreement.
|
||||||
|
|
||||||
|
"""Export the Python grammar and symbols."""
|
||||||
|
|
||||||
|
# Python imports
|
||||||
|
import os
|
||||||
|
|
||||||
|
# Local imports
|
||||||
|
from .pgen2 import token
|
||||||
|
from .pgen2 import driver
|
||||||
|
from . import pytree
|
||||||
|
|
||||||
|
# The grammar file
|
||||||
|
_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "Grammar.txt")
|
||||||
|
_PATTERN_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__),
|
||||||
|
"PatternGrammar.txt")
|
||||||
|
|
||||||
|
|
||||||
|
class Symbols(object):
|
||||||
|
|
||||||
|
def __init__(self, grammar):
|
||||||
|
"""Initializer.
|
||||||
|
|
||||||
|
Creates an attribute for each grammar symbol (nonterminal),
|
||||||
|
whose value is the symbol's type (an int >= 256).
|
||||||
|
"""
|
||||||
|
for name, symbol in grammar.symbol2number.items():
|
||||||
|
setattr(self, name, symbol)
|
||||||
|
|
||||||
|
|
||||||
|
python_grammar = driver.load_packaged_grammar("lib2to3", _GRAMMAR_FILE)
|
||||||
|
|
||||||
|
python_symbols = Symbols(python_grammar)
|
||||||
|
|
||||||
|
python_grammar_no_print_statement = python_grammar.copy()
|
||||||
|
del python_grammar_no_print_statement.keywords["print"]
|
||||||
|
|
||||||
|
pattern_grammar = driver.load_packaged_grammar("lib2to3", _PATTERN_GRAMMAR_FILE)
|
||||||
|
pattern_symbols = Symbols(pattern_grammar)
|
||||||
119
blib2to3/pygram.pyi
Normal file
119
blib2to3/pygram.pyi
Normal file
@ -0,0 +1,119 @@
|
|||||||
|
# Stubs for lib2to3.pygram (Python 3.6)
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
from blib2to3.pgen2.grammar import Grammar
|
||||||
|
|
||||||
|
class Symbols:
|
||||||
|
def __init__(self, grammar: Grammar) -> None: ...
|
||||||
|
|
||||||
|
class python_symbols(Symbols):
|
||||||
|
and_expr: int
|
||||||
|
and_test: int
|
||||||
|
annassign: int
|
||||||
|
arglist: int
|
||||||
|
argument: int
|
||||||
|
arith_expr: int
|
||||||
|
assert_stmt: int
|
||||||
|
async_funcdef: int
|
||||||
|
async_stmt: int
|
||||||
|
atom: int
|
||||||
|
augassign: int
|
||||||
|
break_stmt: int
|
||||||
|
classdef: int
|
||||||
|
comp_for: int
|
||||||
|
comp_if: int
|
||||||
|
comp_iter: int
|
||||||
|
comp_op: int
|
||||||
|
comparison: int
|
||||||
|
compound_stmt: int
|
||||||
|
continue_stmt: int
|
||||||
|
decorated: int
|
||||||
|
decorator: int
|
||||||
|
decorators: int
|
||||||
|
del_stmt: int
|
||||||
|
dictsetmaker: int
|
||||||
|
dotted_as_name: int
|
||||||
|
dotted_as_names: int
|
||||||
|
dotted_name: int
|
||||||
|
encoding_decl: int
|
||||||
|
eval_input: int
|
||||||
|
except_clause: int
|
||||||
|
exec_stmt: int
|
||||||
|
expr: int
|
||||||
|
expr_stmt: int
|
||||||
|
exprlist: int
|
||||||
|
factor: int
|
||||||
|
file_input: int
|
||||||
|
flow_stmt: int
|
||||||
|
for_stmt: int
|
||||||
|
funcdef: int
|
||||||
|
global_stmt: int
|
||||||
|
if_stmt: int
|
||||||
|
import_as_name: int
|
||||||
|
import_as_names: int
|
||||||
|
import_from: int
|
||||||
|
import_name: int
|
||||||
|
import_stmt: int
|
||||||
|
lambdef: int
|
||||||
|
listmaker: int
|
||||||
|
not_test: int
|
||||||
|
old_comp_for: int
|
||||||
|
old_comp_if: int
|
||||||
|
old_comp_iter: int
|
||||||
|
old_lambdef: int
|
||||||
|
old_test: int
|
||||||
|
or_test: int
|
||||||
|
parameters: int
|
||||||
|
pass_stmt: int
|
||||||
|
power: int
|
||||||
|
print_stmt: int
|
||||||
|
raise_stmt: int
|
||||||
|
return_stmt: int
|
||||||
|
shift_expr: int
|
||||||
|
simple_stmt: int
|
||||||
|
single_input: int
|
||||||
|
sliceop: int
|
||||||
|
small_stmt: int
|
||||||
|
star_expr: int
|
||||||
|
stmt: int
|
||||||
|
subscript: int
|
||||||
|
subscriptlist: int
|
||||||
|
suite: int
|
||||||
|
term: int
|
||||||
|
test: int
|
||||||
|
testlist: int
|
||||||
|
testlist1: int
|
||||||
|
testlist_gexp: int
|
||||||
|
testlist_safe: int
|
||||||
|
testlist_star_expr: int
|
||||||
|
tfpdef: int
|
||||||
|
tfplist: int
|
||||||
|
tname: int
|
||||||
|
trailer: int
|
||||||
|
try_stmt: int
|
||||||
|
typedargslist: int
|
||||||
|
varargslist: int
|
||||||
|
vfpdef: int
|
||||||
|
vfplist: int
|
||||||
|
vname: int
|
||||||
|
while_stmt: int
|
||||||
|
with_item: int
|
||||||
|
with_stmt: int
|
||||||
|
with_var: int
|
||||||
|
xor_expr: int
|
||||||
|
yield_arg: int
|
||||||
|
yield_expr: int
|
||||||
|
yield_stmt: int
|
||||||
|
|
||||||
|
class pattern_symbols(Symbols):
|
||||||
|
Alternative: int
|
||||||
|
Alternatives: int
|
||||||
|
Details: int
|
||||||
|
Matcher: int
|
||||||
|
NegatedUnit: int
|
||||||
|
Repeater: int
|
||||||
|
Unit: int
|
||||||
|
|
||||||
|
python_grammar: Grammar
|
||||||
|
python_grammar_no_print_statement: Grammar
|
||||||
|
pattern_grammar: Grammar
|
||||||
854
blib2to3/pytree.py
Normal file
854
blib2to3/pytree.py
Normal file
@ -0,0 +1,854 @@
|
|||||||
|
# Copyright 2006 Google, Inc. All Rights Reserved.
|
||||||
|
# Licensed to PSF under a Contributor Agreement.
|
||||||
|
|
||||||
|
"""
|
||||||
|
Python parse tree definitions.
|
||||||
|
|
||||||
|
This is a very concrete parse tree; we need to keep every token and
|
||||||
|
even the comments and whitespace between tokens.
|
||||||
|
|
||||||
|
There's also a pattern matching implementation here.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__author__ = "Guido van Rossum <guido@python.org>"
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from io import StringIO
|
||||||
|
|
||||||
|
HUGE = 0x7FFFFFFF # maximum repeat count, default max
|
||||||
|
|
||||||
|
_type_reprs = {}
|
||||||
|
def type_repr(type_num):
|
||||||
|
global _type_reprs
|
||||||
|
if not _type_reprs:
|
||||||
|
from .pygram import python_symbols
|
||||||
|
# printing tokens is possible but not as useful
|
||||||
|
# from .pgen2 import token // token.__dict__.items():
|
||||||
|
for name, val in python_symbols.__dict__.items():
|
||||||
|
if type(val) == int: _type_reprs[val] = name
|
||||||
|
return _type_reprs.setdefault(type_num, type_num)
|
||||||
|
|
||||||
|
class Base(object):
|
||||||
|
|
||||||
|
"""
|
||||||
|
Abstract base class for Node and Leaf.
|
||||||
|
|
||||||
|
This provides some default functionality and boilerplate using the
|
||||||
|
template pattern.
|
||||||
|
|
||||||
|
A node may be a subnode of at most one parent.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Default values for instance variables
|
||||||
|
type = None # int: token number (< 256) or symbol number (>= 256)
|
||||||
|
parent = None # Parent node pointer, or None
|
||||||
|
children = () # Tuple of subnodes
|
||||||
|
was_changed = False
|
||||||
|
was_checked = False
|
||||||
|
|
||||||
|
def __new__(cls, *args, **kwds):
|
||||||
|
"""Constructor that prevents Base from being instantiated."""
|
||||||
|
assert cls is not Base, "Cannot instantiate Base"
|
||||||
|
return object.__new__(cls)
|
||||||
|
|
||||||
|
def __eq__(self, other):
|
||||||
|
"""
|
||||||
|
Compare two nodes for equality.
|
||||||
|
|
||||||
|
This calls the method _eq().
|
||||||
|
"""
|
||||||
|
if self.__class__ is not other.__class__:
|
||||||
|
return NotImplemented
|
||||||
|
return self._eq(other)
|
||||||
|
|
||||||
|
__hash__ = None # For Py3 compatibility.
|
||||||
|
|
||||||
|
def _eq(self, other):
|
||||||
|
"""
|
||||||
|
Compare two nodes for equality.
|
||||||
|
|
||||||
|
This is called by __eq__ and __ne__. It is only called if the two nodes
|
||||||
|
have the same type. This must be implemented by the concrete subclass.
|
||||||
|
Nodes should be considered equal if they have the same structure,
|
||||||
|
ignoring the prefix string and other context information.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def clone(self):
|
||||||
|
"""
|
||||||
|
Return a cloned (deep) copy of self.
|
||||||
|
|
||||||
|
This must be implemented by the concrete subclass.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def post_order(self):
|
||||||
|
"""
|
||||||
|
Return a post-order iterator for the tree.
|
||||||
|
|
||||||
|
This must be implemented by the concrete subclass.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def pre_order(self):
|
||||||
|
"""
|
||||||
|
Return a pre-order iterator for the tree.
|
||||||
|
|
||||||
|
This must be implemented by the concrete subclass.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def replace(self, new):
|
||||||
|
"""Replace this node with a new one in the parent."""
|
||||||
|
assert self.parent is not None, str(self)
|
||||||
|
assert new is not None
|
||||||
|
if not isinstance(new, list):
|
||||||
|
new = [new]
|
||||||
|
l_children = []
|
||||||
|
found = False
|
||||||
|
for ch in self.parent.children:
|
||||||
|
if ch is self:
|
||||||
|
assert not found, (self.parent.children, self, new)
|
||||||
|
if new is not None:
|
||||||
|
l_children.extend(new)
|
||||||
|
found = True
|
||||||
|
else:
|
||||||
|
l_children.append(ch)
|
||||||
|
assert found, (self.children, self, new)
|
||||||
|
self.parent.changed()
|
||||||
|
self.parent.children = l_children
|
||||||
|
for x in new:
|
||||||
|
x.parent = self.parent
|
||||||
|
self.parent = None
|
||||||
|
|
||||||
|
def get_lineno(self):
|
||||||
|
"""Return the line number which generated the invocant node."""
|
||||||
|
node = self
|
||||||
|
while not isinstance(node, Leaf):
|
||||||
|
if not node.children:
|
||||||
|
return
|
||||||
|
node = node.children[0]
|
||||||
|
return node.lineno
|
||||||
|
|
||||||
|
def changed(self):
|
||||||
|
if self.parent:
|
||||||
|
self.parent.changed()
|
||||||
|
self.was_changed = True
|
||||||
|
|
||||||
|
def remove(self):
|
||||||
|
"""
|
||||||
|
Remove the node from the tree. Returns the position of the node in its
|
||||||
|
parent's children before it was removed.
|
||||||
|
"""
|
||||||
|
if self.parent:
|
||||||
|
for i, node in enumerate(self.parent.children):
|
||||||
|
if node is self:
|
||||||
|
self.parent.changed()
|
||||||
|
del self.parent.children[i]
|
||||||
|
self.parent = None
|
||||||
|
return i
|
||||||
|
|
||||||
|
@property
|
||||||
|
def next_sibling(self):
|
||||||
|
"""
|
||||||
|
The node immediately following the invocant in their parent's children
|
||||||
|
list. If the invocant does not have a next sibling, it is None
|
||||||
|
"""
|
||||||
|
if self.parent is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Can't use index(); we need to test by identity
|
||||||
|
for i, child in enumerate(self.parent.children):
|
||||||
|
if child is self:
|
||||||
|
try:
|
||||||
|
return self.parent.children[i+1]
|
||||||
|
except IndexError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def prev_sibling(self):
|
||||||
|
"""
|
||||||
|
The node immediately preceding the invocant in their parent's children
|
||||||
|
list. If the invocant does not have a previous sibling, it is None.
|
||||||
|
"""
|
||||||
|
if self.parent is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Can't use index(); we need to test by identity
|
||||||
|
for i, child in enumerate(self.parent.children):
|
||||||
|
if child is self:
|
||||||
|
if i == 0:
|
||||||
|
return None
|
||||||
|
return self.parent.children[i-1]
|
||||||
|
|
||||||
|
def leaves(self):
|
||||||
|
for child in self.children:
|
||||||
|
yield from child.leaves()
|
||||||
|
|
||||||
|
def depth(self):
|
||||||
|
if self.parent is None:
|
||||||
|
return 0
|
||||||
|
return 1 + self.parent.depth()
|
||||||
|
|
||||||
|
def get_suffix(self):
|
||||||
|
"""
|
||||||
|
Return the string immediately following the invocant node. This is
|
||||||
|
effectively equivalent to node.next_sibling.prefix
|
||||||
|
"""
|
||||||
|
next_sib = self.next_sibling
|
||||||
|
if next_sib is None:
|
||||||
|
return ""
|
||||||
|
return next_sib.prefix
|
||||||
|
|
||||||
|
if sys.version_info < (3, 0):
|
||||||
|
def __str__(self):
|
||||||
|
return str(self).encode("ascii")
|
||||||
|
|
||||||
|
class Node(Base):
|
||||||
|
|
||||||
|
"""Concrete implementation for interior nodes."""
|
||||||
|
|
||||||
|
def __init__(self,type, children,
|
||||||
|
context=None,
|
||||||
|
prefix=None,
|
||||||
|
fixers_applied=None):
|
||||||
|
"""
|
||||||
|
Initializer.
|
||||||
|
|
||||||
|
Takes a type constant (a symbol number >= 256), a sequence of
|
||||||
|
child nodes, and an optional context keyword argument.
|
||||||
|
|
||||||
|
As a side effect, the parent pointers of the children are updated.
|
||||||
|
"""
|
||||||
|
assert type >= 256, type
|
||||||
|
self.type = type
|
||||||
|
self.children = list(children)
|
||||||
|
for ch in self.children:
|
||||||
|
assert ch.parent is None, repr(ch)
|
||||||
|
ch.parent = self
|
||||||
|
if prefix is not None:
|
||||||
|
self.prefix = prefix
|
||||||
|
if fixers_applied:
|
||||||
|
self.fixers_applied = fixers_applied[:]
|
||||||
|
else:
|
||||||
|
self.fixers_applied = None
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
"""Return a canonical string representation."""
|
||||||
|
return "%s(%s, %r)" % (self.__class__.__name__,
|
||||||
|
type_repr(self.type),
|
||||||
|
self.children)
|
||||||
|
|
||||||
|
def __unicode__(self):
|
||||||
|
"""
|
||||||
|
Return a pretty string representation.
|
||||||
|
|
||||||
|
This reproduces the input source exactly.
|
||||||
|
"""
|
||||||
|
return "".join(map(str, self.children))
|
||||||
|
|
||||||
|
if sys.version_info > (3, 0):
|
||||||
|
__str__ = __unicode__
|
||||||
|
|
||||||
|
def _eq(self, other):
|
||||||
|
"""Compare two nodes for equality."""
|
||||||
|
return (self.type, self.children) == (other.type, other.children)
|
||||||
|
|
||||||
|
def clone(self):
|
||||||
|
"""Return a cloned (deep) copy of self."""
|
||||||
|
return Node(self.type, [ch.clone() for ch in self.children],
|
||||||
|
fixers_applied=self.fixers_applied)
|
||||||
|
|
||||||
|
def post_order(self):
|
||||||
|
"""Return a post-order iterator for the tree."""
|
||||||
|
for child in self.children:
|
||||||
|
yield from child.post_order()
|
||||||
|
yield self
|
||||||
|
|
||||||
|
def pre_order(self):
|
||||||
|
"""Return a pre-order iterator for the tree."""
|
||||||
|
yield self
|
||||||
|
for child in self.children:
|
||||||
|
yield from child.pre_order()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def prefix(self):
|
||||||
|
"""
|
||||||
|
The whitespace and comments preceding this node in the input.
|
||||||
|
"""
|
||||||
|
if not self.children:
|
||||||
|
return ""
|
||||||
|
return self.children[0].prefix
|
||||||
|
|
||||||
|
@prefix.setter
|
||||||
|
def prefix(self, prefix):
|
||||||
|
if self.children:
|
||||||
|
self.children[0].prefix = prefix
|
||||||
|
|
||||||
|
def set_child(self, i, child):
|
||||||
|
"""
|
||||||
|
Equivalent to 'node.children[i] = child'. This method also sets the
|
||||||
|
child's parent attribute appropriately.
|
||||||
|
"""
|
||||||
|
child.parent = self
|
||||||
|
self.children[i].parent = None
|
||||||
|
self.children[i] = child
|
||||||
|
self.changed()
|
||||||
|
|
||||||
|
def insert_child(self, i, child):
|
||||||
|
"""
|
||||||
|
Equivalent to 'node.children.insert(i, child)'. This method also sets
|
||||||
|
the child's parent attribute appropriately.
|
||||||
|
"""
|
||||||
|
child.parent = self
|
||||||
|
self.children.insert(i, child)
|
||||||
|
self.changed()
|
||||||
|
|
||||||
|
def append_child(self, child):
|
||||||
|
"""
|
||||||
|
Equivalent to 'node.children.append(child)'. This method also sets the
|
||||||
|
child's parent attribute appropriately.
|
||||||
|
"""
|
||||||
|
child.parent = self
|
||||||
|
self.children.append(child)
|
||||||
|
self.changed()
|
||||||
|
|
||||||
|
|
||||||
|
class Leaf(Base):
|
||||||
|
|
||||||
|
"""Concrete implementation for leaf nodes."""
|
||||||
|
|
||||||
|
# Default values for instance variables
|
||||||
|
_prefix = "" # Whitespace and comments preceding this token in the input
|
||||||
|
lineno = 0 # Line where this token starts in the input
|
||||||
|
column = 0 # Column where this token tarts in the input
|
||||||
|
|
||||||
|
def __init__(self, type, value,
|
||||||
|
context=None,
|
||||||
|
prefix=None,
|
||||||
|
fixers_applied=[]):
|
||||||
|
"""
|
||||||
|
Initializer.
|
||||||
|
|
||||||
|
Takes a type constant (a token number < 256), a string value, and an
|
||||||
|
optional context keyword argument.
|
||||||
|
"""
|
||||||
|
assert 0 <= type < 256, type
|
||||||
|
if context is not None:
|
||||||
|
self._prefix, (self.lineno, self.column) = context
|
||||||
|
self.type = type
|
||||||
|
self.value = value
|
||||||
|
if prefix is not None:
|
||||||
|
self._prefix = prefix
|
||||||
|
self.fixers_applied = fixers_applied[:]
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
"""Return a canonical string representation."""
|
||||||
|
from .pgen2.token import tok_name
|
||||||
|
return "%s(%s, %r)" % (self.__class__.__name__,
|
||||||
|
tok_name.get(self.type, self.type),
|
||||||
|
self.value)
|
||||||
|
|
||||||
|
def __unicode__(self):
|
||||||
|
"""
|
||||||
|
Return a pretty string representation.
|
||||||
|
|
||||||
|
This reproduces the input source exactly.
|
||||||
|
"""
|
||||||
|
return self.prefix + str(self.value)
|
||||||
|
|
||||||
|
if sys.version_info > (3, 0):
|
||||||
|
__str__ = __unicode__
|
||||||
|
|
||||||
|
def _eq(self, other):
|
||||||
|
"""Compare two nodes for equality."""
|
||||||
|
return (self.type, self.value) == (other.type, other.value)
|
||||||
|
|
||||||
|
def clone(self):
|
||||||
|
"""Return a cloned (deep) copy of self."""
|
||||||
|
return Leaf(self.type, self.value,
|
||||||
|
(self.prefix, (self.lineno, self.column)),
|
||||||
|
fixers_applied=self.fixers_applied)
|
||||||
|
|
||||||
|
def leaves(self):
|
||||||
|
yield self
|
||||||
|
|
||||||
|
def post_order(self):
|
||||||
|
"""Return a post-order iterator for the tree."""
|
||||||
|
yield self
|
||||||
|
|
||||||
|
def pre_order(self):
|
||||||
|
"""Return a pre-order iterator for the tree."""
|
||||||
|
yield self
|
||||||
|
|
||||||
|
@property
|
||||||
|
def prefix(self):
|
||||||
|
"""
|
||||||
|
The whitespace and comments preceding this token in the input.
|
||||||
|
"""
|
||||||
|
return self._prefix
|
||||||
|
|
||||||
|
@prefix.setter
|
||||||
|
def prefix(self, prefix):
|
||||||
|
self.changed()
|
||||||
|
self._prefix = prefix
|
||||||
|
|
||||||
|
def convert(gr, raw_node):
|
||||||
|
"""
|
||||||
|
Convert raw node information to a Node or Leaf instance.
|
||||||
|
|
||||||
|
This is passed to the parser driver which calls it whenever a reduction of a
|
||||||
|
grammar rule produces a new complete node, so that the tree is build
|
||||||
|
strictly bottom-up.
|
||||||
|
"""
|
||||||
|
type, value, context, children = raw_node
|
||||||
|
if children or type in gr.number2symbol:
|
||||||
|
# If there's exactly one child, return that child instead of
|
||||||
|
# creating a new node.
|
||||||
|
if len(children) == 1:
|
||||||
|
return children[0]
|
||||||
|
return Node(type, children, context=context)
|
||||||
|
else:
|
||||||
|
return Leaf(type, value, context=context)
|
||||||
|
|
||||||
|
|
||||||
|
class BasePattern(object):
|
||||||
|
|
||||||
|
"""
|
||||||
|
A pattern is a tree matching pattern.
|
||||||
|
|
||||||
|
It looks for a specific node type (token or symbol), and
|
||||||
|
optionally for a specific content.
|
||||||
|
|
||||||
|
This is an abstract base class. There are three concrete
|
||||||
|
subclasses:
|
||||||
|
|
||||||
|
- LeafPattern matches a single leaf node;
|
||||||
|
- NodePattern matches a single node (usually non-leaf);
|
||||||
|
- WildcardPattern matches a sequence of nodes of variable length.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Defaults for instance variables
|
||||||
|
type = None # Node type (token if < 256, symbol if >= 256)
|
||||||
|
content = None # Optional content matching pattern
|
||||||
|
name = None # Optional name used to store match in results dict
|
||||||
|
|
||||||
|
def __new__(cls, *args, **kwds):
|
||||||
|
"""Constructor that prevents BasePattern from being instantiated."""
|
||||||
|
assert cls is not BasePattern, "Cannot instantiate BasePattern"
|
||||||
|
return object.__new__(cls)
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
args = [type_repr(self.type), self.content, self.name]
|
||||||
|
while args and args[-1] is None:
|
||||||
|
del args[-1]
|
||||||
|
return "%s(%s)" % (self.__class__.__name__, ", ".join(map(repr, args)))
|
||||||
|
|
||||||
|
def optimize(self):
|
||||||
|
"""
|
||||||
|
A subclass can define this as a hook for optimizations.
|
||||||
|
|
||||||
|
Returns either self or another node with the same effect.
|
||||||
|
"""
|
||||||
|
return self
|
||||||
|
|
||||||
|
def match(self, node, results=None):
|
||||||
|
"""
|
||||||
|
Does this pattern exactly match a node?
|
||||||
|
|
||||||
|
Returns True if it matches, False if not.
|
||||||
|
|
||||||
|
If results is not None, it must be a dict which will be
|
||||||
|
updated with the nodes matching named subpatterns.
|
||||||
|
|
||||||
|
Default implementation for non-wildcard patterns.
|
||||||
|
"""
|
||||||
|
if self.type is not None and node.type != self.type:
|
||||||
|
return False
|
||||||
|
if self.content is not None:
|
||||||
|
r = None
|
||||||
|
if results is not None:
|
||||||
|
r = {}
|
||||||
|
if not self._submatch(node, r):
|
||||||
|
return False
|
||||||
|
if r:
|
||||||
|
results.update(r)
|
||||||
|
if results is not None and self.name:
|
||||||
|
results[self.name] = node
|
||||||
|
return True
|
||||||
|
|
||||||
|
def match_seq(self, nodes, results=None):
|
||||||
|
"""
|
||||||
|
Does this pattern exactly match a sequence of nodes?
|
||||||
|
|
||||||
|
Default implementation for non-wildcard patterns.
|
||||||
|
"""
|
||||||
|
if len(nodes) != 1:
|
||||||
|
return False
|
||||||
|
return self.match(nodes[0], results)
|
||||||
|
|
||||||
|
def generate_matches(self, nodes):
|
||||||
|
"""
|
||||||
|
Generator yielding all matches for this pattern.
|
||||||
|
|
||||||
|
Default implementation for non-wildcard patterns.
|
||||||
|
"""
|
||||||
|
r = {}
|
||||||
|
if nodes and self.match(nodes[0], r):
|
||||||
|
yield 1, r
|
||||||
|
|
||||||
|
|
||||||
|
class LeafPattern(BasePattern):
|
||||||
|
|
||||||
|
def __init__(self, type=None, content=None, name=None):
|
||||||
|
"""
|
||||||
|
Initializer. Takes optional type, content, and name.
|
||||||
|
|
||||||
|
The type, if given must be a token type (< 256). If not given,
|
||||||
|
this matches any *leaf* node; the content may still be required.
|
||||||
|
|
||||||
|
The content, if given, must be a string.
|
||||||
|
|
||||||
|
If a name is given, the matching node is stored in the results
|
||||||
|
dict under that key.
|
||||||
|
"""
|
||||||
|
if type is not None:
|
||||||
|
assert 0 <= type < 256, type
|
||||||
|
if content is not None:
|
||||||
|
assert isinstance(content, str), repr(content)
|
||||||
|
self.type = type
|
||||||
|
self.content = content
|
||||||
|
self.name = name
|
||||||
|
|
||||||
|
def match(self, node, results=None):
|
||||||
|
"""Override match() to insist on a leaf node."""
|
||||||
|
if not isinstance(node, Leaf):
|
||||||
|
return False
|
||||||
|
return BasePattern.match(self, node, results)
|
||||||
|
|
||||||
|
def _submatch(self, node, results=None):
|
||||||
|
"""
|
||||||
|
Match the pattern's content to the node's children.
|
||||||
|
|
||||||
|
This assumes the node type matches and self.content is not None.
|
||||||
|
|
||||||
|
Returns True if it matches, False if not.
|
||||||
|
|
||||||
|
If results is not None, it must be a dict which will be
|
||||||
|
updated with the nodes matching named subpatterns.
|
||||||
|
|
||||||
|
When returning False, the results dict may still be updated.
|
||||||
|
"""
|
||||||
|
return self.content == node.value
|
||||||
|
|
||||||
|
|
||||||
|
class NodePattern(BasePattern):
|
||||||
|
|
||||||
|
wildcards = False
|
||||||
|
|
||||||
|
def __init__(self, type=None, content=None, name=None):
|
||||||
|
"""
|
||||||
|
Initializer. Takes optional type, content, and name.
|
||||||
|
|
||||||
|
The type, if given, must be a symbol type (>= 256). If the
|
||||||
|
type is None this matches *any* single node (leaf or not),
|
||||||
|
except if content is not None, in which it only matches
|
||||||
|
non-leaf nodes that also match the content pattern.
|
||||||
|
|
||||||
|
The content, if not None, must be a sequence of Patterns that
|
||||||
|
must match the node's children exactly. If the content is
|
||||||
|
given, the type must not be None.
|
||||||
|
|
||||||
|
If a name is given, the matching node is stored in the results
|
||||||
|
dict under that key.
|
||||||
|
"""
|
||||||
|
if type is not None:
|
||||||
|
assert type >= 256, type
|
||||||
|
if content is not None:
|
||||||
|
assert not isinstance(content, str), repr(content)
|
||||||
|
content = list(content)
|
||||||
|
for i, item in enumerate(content):
|
||||||
|
assert isinstance(item, BasePattern), (i, item)
|
||||||
|
if isinstance(item, WildcardPattern):
|
||||||
|
self.wildcards = True
|
||||||
|
self.type = type
|
||||||
|
self.content = content
|
||||||
|
self.name = name
|
||||||
|
|
||||||
|
def _submatch(self, node, results=None):
|
||||||
|
"""
|
||||||
|
Match the pattern's content to the node's children.
|
||||||
|
|
||||||
|
This assumes the node type matches and self.content is not None.
|
||||||
|
|
||||||
|
Returns True if it matches, False if not.
|
||||||
|
|
||||||
|
If results is not None, it must be a dict which will be
|
||||||
|
updated with the nodes matching named subpatterns.
|
||||||
|
|
||||||
|
When returning False, the results dict may still be updated.
|
||||||
|
"""
|
||||||
|
if self.wildcards:
|
||||||
|
for c, r in generate_matches(self.content, node.children):
|
||||||
|
if c == len(node.children):
|
||||||
|
if results is not None:
|
||||||
|
results.update(r)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
if len(self.content) != len(node.children):
|
||||||
|
return False
|
||||||
|
for subpattern, child in zip(self.content, node.children):
|
||||||
|
if not subpattern.match(child, results):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class WildcardPattern(BasePattern):
|
||||||
|
|
||||||
|
"""
|
||||||
|
A wildcard pattern can match zero or more nodes.
|
||||||
|
|
||||||
|
This has all the flexibility needed to implement patterns like:
|
||||||
|
|
||||||
|
.* .+ .? .{m,n}
|
||||||
|
(a b c | d e | f)
|
||||||
|
(...)* (...)+ (...)? (...){m,n}
|
||||||
|
|
||||||
|
except it always uses non-greedy matching.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, content=None, min=0, max=HUGE, name=None):
|
||||||
|
"""
|
||||||
|
Initializer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
content: optional sequence of subsequences of patterns;
|
||||||
|
if absent, matches one node;
|
||||||
|
if present, each subsequence is an alternative [*]
|
||||||
|
min: optional minimum number of times to match, default 0
|
||||||
|
max: optional maximum number of times to match, default HUGE
|
||||||
|
name: optional name assigned to this match
|
||||||
|
|
||||||
|
[*] Thus, if content is [[a, b, c], [d, e], [f, g, h]] this is
|
||||||
|
equivalent to (a b c | d e | f g h); if content is None,
|
||||||
|
this is equivalent to '.' in regular expression terms.
|
||||||
|
The min and max parameters work as follows:
|
||||||
|
min=0, max=maxint: .*
|
||||||
|
min=1, max=maxint: .+
|
||||||
|
min=0, max=1: .?
|
||||||
|
min=1, max=1: .
|
||||||
|
If content is not None, replace the dot with the parenthesized
|
||||||
|
list of alternatives, e.g. (a b c | d e | f g h)*
|
||||||
|
"""
|
||||||
|
assert 0 <= min <= max <= HUGE, (min, max)
|
||||||
|
if content is not None:
|
||||||
|
content = tuple(map(tuple, content)) # Protect against alterations
|
||||||
|
# Check sanity of alternatives
|
||||||
|
assert len(content), repr(content) # Can't have zero alternatives
|
||||||
|
for alt in content:
|
||||||
|
assert len(alt), repr(alt) # Can have empty alternatives
|
||||||
|
self.content = content
|
||||||
|
self.min = min
|
||||||
|
self.max = max
|
||||||
|
self.name = name
|
||||||
|
|
||||||
|
def optimize(self):
|
||||||
|
"""Optimize certain stacked wildcard patterns."""
|
||||||
|
subpattern = None
|
||||||
|
if (self.content is not None and
|
||||||
|
len(self.content) == 1 and len(self.content[0]) == 1):
|
||||||
|
subpattern = self.content[0][0]
|
||||||
|
if self.min == 1 and self.max == 1:
|
||||||
|
if self.content is None:
|
||||||
|
return NodePattern(name=self.name)
|
||||||
|
if subpattern is not None and self.name == subpattern.name:
|
||||||
|
return subpattern.optimize()
|
||||||
|
if (self.min <= 1 and isinstance(subpattern, WildcardPattern) and
|
||||||
|
subpattern.min <= 1 and self.name == subpattern.name):
|
||||||
|
return WildcardPattern(subpattern.content,
|
||||||
|
self.min*subpattern.min,
|
||||||
|
self.max*subpattern.max,
|
||||||
|
subpattern.name)
|
||||||
|
return self
|
||||||
|
|
||||||
|
def match(self, node, results=None):
|
||||||
|
"""Does this pattern exactly match a node?"""
|
||||||
|
return self.match_seq([node], results)
|
||||||
|
|
||||||
|
def match_seq(self, nodes, results=None):
|
||||||
|
"""Does this pattern exactly match a sequence of nodes?"""
|
||||||
|
for c, r in self.generate_matches(nodes):
|
||||||
|
if c == len(nodes):
|
||||||
|
if results is not None:
|
||||||
|
results.update(r)
|
||||||
|
if self.name:
|
||||||
|
results[self.name] = list(nodes)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def generate_matches(self, nodes):
|
||||||
|
"""
|
||||||
|
Generator yielding matches for a sequence of nodes.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
nodes: sequence of nodes
|
||||||
|
|
||||||
|
Yields:
|
||||||
|
(count, results) tuples where:
|
||||||
|
count: the match comprises nodes[:count];
|
||||||
|
results: dict containing named submatches.
|
||||||
|
"""
|
||||||
|
if self.content is None:
|
||||||
|
# Shortcut for special case (see __init__.__doc__)
|
||||||
|
for count in range(self.min, 1 + min(len(nodes), self.max)):
|
||||||
|
r = {}
|
||||||
|
if self.name:
|
||||||
|
r[self.name] = nodes[:count]
|
||||||
|
yield count, r
|
||||||
|
elif self.name == "bare_name":
|
||||||
|
yield self._bare_name_matches(nodes)
|
||||||
|
else:
|
||||||
|
# The reason for this is that hitting the recursion limit usually
|
||||||
|
# results in some ugly messages about how RuntimeErrors are being
|
||||||
|
# ignored. We only have to do this on CPython, though, because other
|
||||||
|
# implementations don't have this nasty bug in the first place.
|
||||||
|
if hasattr(sys, "getrefcount"):
|
||||||
|
save_stderr = sys.stderr
|
||||||
|
sys.stderr = StringIO()
|
||||||
|
try:
|
||||||
|
for count, r in self._recursive_matches(nodes, 0):
|
||||||
|
if self.name:
|
||||||
|
r[self.name] = nodes[:count]
|
||||||
|
yield count, r
|
||||||
|
except RuntimeError:
|
||||||
|
# We fall back to the iterative pattern matching scheme if the recursive
|
||||||
|
# scheme hits the recursion limit.
|
||||||
|
for count, r in self._iterative_matches(nodes):
|
||||||
|
if self.name:
|
||||||
|
r[self.name] = nodes[:count]
|
||||||
|
yield count, r
|
||||||
|
finally:
|
||||||
|
if hasattr(sys, "getrefcount"):
|
||||||
|
sys.stderr = save_stderr
|
||||||
|
|
||||||
|
def _iterative_matches(self, nodes):
|
||||||
|
"""Helper to iteratively yield the matches."""
|
||||||
|
nodelen = len(nodes)
|
||||||
|
if 0 >= self.min:
|
||||||
|
yield 0, {}
|
||||||
|
|
||||||
|
results = []
|
||||||
|
# generate matches that use just one alt from self.content
|
||||||
|
for alt in self.content:
|
||||||
|
for c, r in generate_matches(alt, nodes):
|
||||||
|
yield c, r
|
||||||
|
results.append((c, r))
|
||||||
|
|
||||||
|
# for each match, iterate down the nodes
|
||||||
|
while results:
|
||||||
|
new_results = []
|
||||||
|
for c0, r0 in results:
|
||||||
|
# stop if the entire set of nodes has been matched
|
||||||
|
if c0 < nodelen and c0 <= self.max:
|
||||||
|
for alt in self.content:
|
||||||
|
for c1, r1 in generate_matches(alt, nodes[c0:]):
|
||||||
|
if c1 > 0:
|
||||||
|
r = {}
|
||||||
|
r.update(r0)
|
||||||
|
r.update(r1)
|
||||||
|
yield c0 + c1, r
|
||||||
|
new_results.append((c0 + c1, r))
|
||||||
|
results = new_results
|
||||||
|
|
||||||
|
def _bare_name_matches(self, nodes):
|
||||||
|
"""Special optimized matcher for bare_name."""
|
||||||
|
count = 0
|
||||||
|
r = {}
|
||||||
|
done = False
|
||||||
|
max = len(nodes)
|
||||||
|
while not done and count < max:
|
||||||
|
done = True
|
||||||
|
for leaf in self.content:
|
||||||
|
if leaf[0].match(nodes[count], r):
|
||||||
|
count += 1
|
||||||
|
done = False
|
||||||
|
break
|
||||||
|
r[self.name] = nodes[:count]
|
||||||
|
return count, r
|
||||||
|
|
||||||
|
def _recursive_matches(self, nodes, count):
|
||||||
|
"""Helper to recursively yield the matches."""
|
||||||
|
assert self.content is not None
|
||||||
|
if count >= self.min:
|
||||||
|
yield 0, {}
|
||||||
|
if count < self.max:
|
||||||
|
for alt in self.content:
|
||||||
|
for c0, r0 in generate_matches(alt, nodes):
|
||||||
|
for c1, r1 in self._recursive_matches(nodes[c0:], count+1):
|
||||||
|
r = {}
|
||||||
|
r.update(r0)
|
||||||
|
r.update(r1)
|
||||||
|
yield c0 + c1, r
|
||||||
|
|
||||||
|
|
||||||
|
class NegatedPattern(BasePattern):
|
||||||
|
|
||||||
|
def __init__(self, content=None):
|
||||||
|
"""
|
||||||
|
Initializer.
|
||||||
|
|
||||||
|
The argument is either a pattern or None. If it is None, this
|
||||||
|
only matches an empty sequence (effectively '$' in regex
|
||||||
|
lingo). If it is not None, this matches whenever the argument
|
||||||
|
pattern doesn't have any matches.
|
||||||
|
"""
|
||||||
|
if content is not None:
|
||||||
|
assert isinstance(content, BasePattern), repr(content)
|
||||||
|
self.content = content
|
||||||
|
|
||||||
|
def match(self, node):
|
||||||
|
# We never match a node in its entirety
|
||||||
|
return False
|
||||||
|
|
||||||
|
def match_seq(self, nodes):
|
||||||
|
# We only match an empty sequence of nodes in its entirety
|
||||||
|
return len(nodes) == 0
|
||||||
|
|
||||||
|
def generate_matches(self, nodes):
|
||||||
|
if self.content is None:
|
||||||
|
# Return a match if there is an empty sequence
|
||||||
|
if len(nodes) == 0:
|
||||||
|
yield 0, {}
|
||||||
|
else:
|
||||||
|
# Return a match if the argument pattern has no matches
|
||||||
|
for c, r in self.content.generate_matches(nodes):
|
||||||
|
return
|
||||||
|
yield 0, {}
|
||||||
|
|
||||||
|
|
||||||
|
def generate_matches(patterns, nodes):
|
||||||
|
"""
|
||||||
|
Generator yielding matches for a sequence of patterns and nodes.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
patterns: a sequence of patterns
|
||||||
|
nodes: a sequence of nodes
|
||||||
|
|
||||||
|
Yields:
|
||||||
|
(count, results) tuples where:
|
||||||
|
count: the entire sequence of patterns matches nodes[:count];
|
||||||
|
results: dict containing named submatches.
|
||||||
|
"""
|
||||||
|
if not patterns:
|
||||||
|
yield 0, {}
|
||||||
|
else:
|
||||||
|
p, rest = patterns[0], patterns[1:]
|
||||||
|
for c0, r0 in p.generate_matches(nodes):
|
||||||
|
if not rest:
|
||||||
|
yield c0, r0
|
||||||
|
else:
|
||||||
|
for c1, r1 in generate_matches(rest, nodes[c0:]):
|
||||||
|
r = {}
|
||||||
|
r.update(r0)
|
||||||
|
r.update(r1)
|
||||||
|
yield c0 + c1, r
|
||||||
86
blib2to3/pytree.pyi
Normal file
86
blib2to3/pytree.pyi
Normal file
@ -0,0 +1,86 @@
|
|||||||
|
# Stubs for lib2to3.pytree (Python 3.6)
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from typing import Any, Callable, Dict, Iterator, List, Optional, Text, Tuple, TypeVar, Union
|
||||||
|
|
||||||
|
from blib2to3.pgen2.grammar import Grammar
|
||||||
|
|
||||||
|
_P = TypeVar('_P')
|
||||||
|
_NL = Union[Node, Leaf]
|
||||||
|
_Context = Tuple[Text, int, int]
|
||||||
|
_Results = Dict[Text, _NL]
|
||||||
|
_RawNode = Tuple[int, Text, _Context, Optional[List[_NL]]]
|
||||||
|
_Convert = Callable[[Grammar, _RawNode], Any]
|
||||||
|
|
||||||
|
HUGE: int
|
||||||
|
|
||||||
|
def type_repr(type_num: int) -> Text: ...
|
||||||
|
|
||||||
|
class Base:
|
||||||
|
type: int
|
||||||
|
parent: Optional[Node]
|
||||||
|
prefix: Text
|
||||||
|
children: List[_NL]
|
||||||
|
was_changed: bool
|
||||||
|
was_checked: bool
|
||||||
|
def __eq__(self, other: Any) -> bool: ...
|
||||||
|
def _eq(self: _P, other: _P) -> bool: ...
|
||||||
|
def clone(self: _P) -> _P: ...
|
||||||
|
def post_order(self) -> Iterator[_NL]: ...
|
||||||
|
def pre_order(self) -> Iterator[_NL]: ...
|
||||||
|
def replace(self, new: Union[_NL, List[_NL]]) -> None: ...
|
||||||
|
def get_lineno(self) -> int: ...
|
||||||
|
def changed(self) -> None: ...
|
||||||
|
def remove(self) -> Optional[int]: ...
|
||||||
|
@property
|
||||||
|
def next_sibling(self) -> Optional[_NL]: ...
|
||||||
|
@property
|
||||||
|
def prev_sibling(self) -> Optional[_NL]: ...
|
||||||
|
def leaves(self) -> Iterator[Leaf]: ...
|
||||||
|
def depth(self) -> int: ...
|
||||||
|
def get_suffix(self) -> Text: ...
|
||||||
|
if sys.version_info < (3,):
|
||||||
|
def get_prefix(self) -> Text: ...
|
||||||
|
def set_prefix(self, prefix: Text) -> None: ...
|
||||||
|
|
||||||
|
class Node(Base):
|
||||||
|
fixers_applied: List[Any]
|
||||||
|
def __init__(self, type: int, children: List[_NL], context: Optional[Any] = ..., prefix: Optional[Text] = ..., fixers_applied: Optional[List[Any]] = ...) -> None: ...
|
||||||
|
def set_child(self, i: int, child: _NL) -> None: ...
|
||||||
|
def insert_child(self, i: int, child: _NL) -> None: ...
|
||||||
|
def append_child(self, child: _NL) -> None: ...
|
||||||
|
|
||||||
|
class Leaf(Base):
|
||||||
|
lineno: int
|
||||||
|
column: int
|
||||||
|
value: Text
|
||||||
|
fixers_applied: List[Any]
|
||||||
|
def __init__(self, type: int, value: Text, context: Optional[_Context] = ..., prefix: Optional[Text] = ..., fixers_applied: List[Any] = ...) -> None: ...
|
||||||
|
|
||||||
|
def convert(gr: Grammar, raw_node: _RawNode) -> _NL: ...
|
||||||
|
|
||||||
|
class BasePattern:
|
||||||
|
type: int
|
||||||
|
content: Optional[Text]
|
||||||
|
name: Optional[Text]
|
||||||
|
def optimize(self) -> BasePattern: ... # sic, subclasses are free to optimize themselves into different patterns
|
||||||
|
def match(self, node: _NL, results: Optional[_Results] = ...) -> bool: ...
|
||||||
|
def match_seq(self, nodes: List[_NL], results: Optional[_Results] = ...) -> bool: ...
|
||||||
|
def generate_matches(self, nodes: List[_NL]) -> Iterator[Tuple[int, _Results]]: ...
|
||||||
|
|
||||||
|
class LeafPattern(BasePattern):
|
||||||
|
def __init__(self, type: Optional[int] = ..., content: Optional[Text] = ..., name: Optional[Text] = ...) -> None: ...
|
||||||
|
|
||||||
|
class NodePattern(BasePattern):
|
||||||
|
wildcards: bool
|
||||||
|
def __init__(self, type: Optional[int] = ..., content: Optional[Text] = ..., name: Optional[Text] = ...) -> None: ...
|
||||||
|
|
||||||
|
class WildcardPattern(BasePattern):
|
||||||
|
min: int
|
||||||
|
max: int
|
||||||
|
def __init__(self, content: Optional[Text] = ..., min: int = ..., max: int = ..., name: Optional[Text] = ...) -> None: ...
|
||||||
|
|
||||||
|
class NegatedPattern(BasePattern):
|
||||||
|
def __init__(self, content: Optional[Text] = ...) -> None: ...
|
||||||
|
|
||||||
|
def generate_matches(patterns: List[BasePattern], nodes: List[_NL]) -> Iterator[Tuple[int, _Results]]: ...
|
||||||
31
mypy.ini
Normal file
31
mypy.ini
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
[mypy]
|
||||||
|
# Specify the target platform details in config, so your developers are
|
||||||
|
# free to run mypy on Windows, Linux, or macOS and get consistent
|
||||||
|
# results.
|
||||||
|
python_version=3.6
|
||||||
|
platform=linux
|
||||||
|
|
||||||
|
# flake8-mypy expects the two following for sensible formatting
|
||||||
|
show_column_numbers=True
|
||||||
|
|
||||||
|
# show error messages from unrelated files
|
||||||
|
follow_imports=normal
|
||||||
|
|
||||||
|
# suppress errors about unsatisfied imports
|
||||||
|
ignore_missing_imports=True
|
||||||
|
|
||||||
|
# be strict
|
||||||
|
disallow_untyped_calls=True
|
||||||
|
warn_return_any=True
|
||||||
|
strict_optional=True
|
||||||
|
warn_no_return=True
|
||||||
|
warn_redundant_casts=True
|
||||||
|
warn_unused_ignores=True
|
||||||
|
|
||||||
|
# The following are off by default. Flip them on if you feel
|
||||||
|
# adventurous.
|
||||||
|
disallow_untyped_defs=True
|
||||||
|
check_untyped_defs=True
|
||||||
|
|
||||||
|
# No incremental mode
|
||||||
|
cache_dir=/dev/null
|
||||||
67
setup.py
Normal file
67
setup.py
Normal file
@ -0,0 +1,67 @@
|
|||||||
|
# Copyright (C) 2018 Łukasz Langa
|
||||||
|
import ast
|
||||||
|
import re
|
||||||
|
from setuptools import setup
|
||||||
|
import sys
|
||||||
|
|
||||||
|
assert sys.version_info >= (3, 6, 0), "black requires Python 3.6+"
|
||||||
|
from pathlib import Path # noqa E402
|
||||||
|
|
||||||
|
CURRENT_DIR = Path(__file__).parent
|
||||||
|
|
||||||
|
|
||||||
|
def get_long_description():
|
||||||
|
readme_md = CURRENT_DIR / 'README.md'
|
||||||
|
try:
|
||||||
|
import pypandoc
|
||||||
|
return pypandoc.convert_file(str(readme_md), 'rst')
|
||||||
|
|
||||||
|
except (IOError, ImportError):
|
||||||
|
print()
|
||||||
|
print(
|
||||||
|
'\x1b[31m\x1b[1mwarning:\x1b[0m\x1b[31m pandoc not found, '
|
||||||
|
'long description will be ugly (PyPI does not support .md).'
|
||||||
|
'\x1b[0m'
|
||||||
|
)
|
||||||
|
print()
|
||||||
|
with open(readme_md, encoding='utf8') as ld_file:
|
||||||
|
return ld_file.read()
|
||||||
|
|
||||||
|
|
||||||
|
def get_version():
|
||||||
|
black_py = CURRENT_DIR / 'black.py'
|
||||||
|
_version_re = re.compile(r'__version__\s+=\s+(?P<version>.*)')
|
||||||
|
with open(black_py, 'r', encoding='utf8') as f:
|
||||||
|
version = _version_re.search(f.read()).group('version')
|
||||||
|
return str(ast.literal_eval(version))
|
||||||
|
|
||||||
|
|
||||||
|
setup(
|
||||||
|
name='black',
|
||||||
|
version=get_version(),
|
||||||
|
description="The uncompromising code formatter.",
|
||||||
|
long_description=get_long_description(),
|
||||||
|
keywords='automation formatter yapf autopep8 pyfmt gofmt rustfmt',
|
||||||
|
author='Łukasz Langa',
|
||||||
|
author_email='lukasz@langa.pl',
|
||||||
|
url='https://github.com/ambv/black',
|
||||||
|
license='MIT',
|
||||||
|
py_modules=['black'],
|
||||||
|
packages=['blib2to3', 'blib2to3.pgen2'],
|
||||||
|
python_requires=">=3.6",
|
||||||
|
zip_safe=False,
|
||||||
|
install_requires=['click', 'attrs'],
|
||||||
|
test_suite='tests.test_black',
|
||||||
|
classifiers=[
|
||||||
|
'Development Status :: 3 - Alpha',
|
||||||
|
'Environment :: Console',
|
||||||
|
'Intended Audience :: Developers',
|
||||||
|
'License :: OSI Approved :: MIT License',
|
||||||
|
'Operating System :: OS Independent',
|
||||||
|
'Programming Language :: Python',
|
||||||
|
'Programming Language :: Python :: 3.6',
|
||||||
|
'Topic :: Software Development :: Libraries :: Python Modules',
|
||||||
|
'Topic :: Software Development :: Quality Assurance',
|
||||||
|
],
|
||||||
|
entry_points={'console_scripts': ['black=black:main']},
|
||||||
|
)
|
||||||
8
tests/.flake8
Normal file
8
tests/.flake8
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
# Like the base Black .flake8 but also ignores F811 which is used deliberately
|
||||||
|
# in test files.
|
||||||
|
|
||||||
|
[flake8]
|
||||||
|
ignore = E266, E501, F811
|
||||||
|
max-line-length = 80
|
||||||
|
max-complexity = 12
|
||||||
|
select = B,C,E,F,W,T4,B9
|
||||||
27
tests/cantfit.py
Normal file
27
tests/cantfit.py
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
# long variable name
|
||||||
|
this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it = 0
|
||||||
|
this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it = 1 # with a comment
|
||||||
|
this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it = [
|
||||||
|
1, 2, 3
|
||||||
|
]
|
||||||
|
this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it = function()
|
||||||
|
this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it = function(
|
||||||
|
arg1, arg2, arg3
|
||||||
|
)
|
||||||
|
this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it = function(
|
||||||
|
[1, 2, 3], arg1, [1, 2, 3], arg2, [1, 2, 3], arg3
|
||||||
|
)
|
||||||
|
# long function name
|
||||||
|
normal_name = but_the_function_name_is_now_ridiculously_long_and_it_is_still_super_annoying()
|
||||||
|
normal_name = but_the_function_name_is_now_ridiculously_long_and_it_is_still_super_annoying(
|
||||||
|
arg1, arg2, arg3
|
||||||
|
)
|
||||||
|
normal_name = but_the_function_name_is_now_ridiculously_long_and_it_is_still_super_annoying(
|
||||||
|
[1, 2, 3], arg1, [1, 2, 3], arg2, [1, 2, 3], arg3
|
||||||
|
)
|
||||||
|
# long arguments
|
||||||
|
normal_name = normal_function_name(
|
||||||
|
"but with super long string arguments that on their own exceed the line limit so there's no way it can ever fit",
|
||||||
|
"eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs",
|
||||||
|
this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0,
|
||||||
|
)
|
||||||
60
tests/comments.py
Normal file
60
tests/comments.py
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# Some license here.
|
||||||
|
#
|
||||||
|
# Has many lines. Many, many lines.
|
||||||
|
# Many, many, many lines.
|
||||||
|
"""Module docstring.
|
||||||
|
|
||||||
|
Possibly also many, many lines.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os.path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import a
|
||||||
|
from b.c import X # some noqa comment
|
||||||
|
|
||||||
|
try:
|
||||||
|
import fast
|
||||||
|
except ImportError:
|
||||||
|
import slow as fast
|
||||||
|
|
||||||
|
|
||||||
|
# Some comment before a function.
|
||||||
|
def function(default=None):
|
||||||
|
"""Docstring comes first.
|
||||||
|
|
||||||
|
Possibly many lines.
|
||||||
|
"""
|
||||||
|
# FIXME: Some comment about why this function is crap but still in production.
|
||||||
|
import inner_imports
|
||||||
|
|
||||||
|
if inner_imports.are_evil():
|
||||||
|
# Explains why we have this if.
|
||||||
|
# In great detail indeed.
|
||||||
|
x = X()
|
||||||
|
return x.method1() # type: ignore
|
||||||
|
|
||||||
|
# This return is also commented for some reason.
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
# Explains why we use global state.
|
||||||
|
GLOBAL_STATE = {'a': a(1), 'b': a(2), 'c': a(3)}
|
||||||
|
|
||||||
|
|
||||||
|
# Another comment
|
||||||
|
@fast(really=True)
|
||||||
|
async def wat():
|
||||||
|
async with X.open_async() as x: # Some more comments
|
||||||
|
result = await x.method1()
|
||||||
|
# Comment after ending a block.
|
||||||
|
if result:
|
||||||
|
print('A OK', file=sys.stdout)
|
||||||
|
# Comment between things.
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
# Some closing comments.
|
||||||
|
# Maybe Vim or Emacs directives for formatting.
|
||||||
|
# Who knows.
|
||||||
202
tests/comments2.py
Normal file
202
tests/comments2.py
Normal file
@ -0,0 +1,202 @@
|
|||||||
|
# Please keep __all__ alphabetized within each category.
|
||||||
|
__all__ = [
|
||||||
|
# Super-special typing primitives.
|
||||||
|
'Any',
|
||||||
|
'Callable',
|
||||||
|
'ClassVar',
|
||||||
|
|
||||||
|
# ABCs (from collections.abc).
|
||||||
|
'AbstractSet', # collections.abc.Set.
|
||||||
|
'ByteString',
|
||||||
|
'Container',
|
||||||
|
|
||||||
|
# Concrete collection types.
|
||||||
|
'Counter',
|
||||||
|
'Deque',
|
||||||
|
'Dict',
|
||||||
|
'DefaultDict',
|
||||||
|
'List',
|
||||||
|
'Set',
|
||||||
|
'FrozenSet',
|
||||||
|
'NamedTuple', # Not really a type.
|
||||||
|
'Generator',
|
||||||
|
]
|
||||||
|
|
||||||
|
def inline_comments_in_brackets_ruin_everything():
|
||||||
|
if typedargslist:
|
||||||
|
parameters.children = [
|
||||||
|
parameters.children[0], # (1
|
||||||
|
body,
|
||||||
|
parameters.children[-1], # )1
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
parameters.children = [
|
||||||
|
parameters.children[0], # (2 what if this was actually long
|
||||||
|
body,
|
||||||
|
parameters.children[-1], # )2
|
||||||
|
]
|
||||||
|
if (self._proc is not None and
|
||||||
|
# has the child process finished?
|
||||||
|
self._returncode is None and
|
||||||
|
# the child process has finished, but the
|
||||||
|
# transport hasn't been notified yet?
|
||||||
|
self._proc.poll() is None):
|
||||||
|
pass
|
||||||
|
short = [
|
||||||
|
# one
|
||||||
|
1,
|
||||||
|
# two
|
||||||
|
2]
|
||||||
|
call(arg1, arg2, """
|
||||||
|
short
|
||||||
|
""", arg3=True)
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
call2(
|
||||||
|
#short
|
||||||
|
arg1,
|
||||||
|
#but
|
||||||
|
arg2,
|
||||||
|
#multiline
|
||||||
|
"""
|
||||||
|
short
|
||||||
|
""",
|
||||||
|
# yup
|
||||||
|
arg3=True)
|
||||||
|
lcomp = [
|
||||||
|
element # yup
|
||||||
|
for element in collection # yup
|
||||||
|
if element is not None # right
|
||||||
|
]
|
||||||
|
lcomp2 = [
|
||||||
|
# hello
|
||||||
|
element
|
||||||
|
# yup
|
||||||
|
for element in collection
|
||||||
|
# right
|
||||||
|
if element is not None
|
||||||
|
]
|
||||||
|
lcomp3 = [
|
||||||
|
# This one is actually too long to fit in a single line.
|
||||||
|
element.split('\n', 1)[0]
|
||||||
|
# yup
|
||||||
|
for element in collection.select_elements()
|
||||||
|
# right
|
||||||
|
if element is not None
|
||||||
|
]
|
||||||
|
return Node(
|
||||||
|
syms.simple_stmt,
|
||||||
|
[
|
||||||
|
Node(statement, result),
|
||||||
|
Leaf(token.NEWLINE, '\n'), # FIXME: \r\n?
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
instruction()
|
||||||
|
|
||||||
|
# END COMMENTS
|
||||||
|
# MORE END COMMENTS
|
||||||
|
|
||||||
|
|
||||||
|
# output
|
||||||
|
|
||||||
|
|
||||||
|
# Please keep __all__ alphabetized within each category.
|
||||||
|
__all__ = [
|
||||||
|
# Super-special typing primitives.
|
||||||
|
'Any',
|
||||||
|
'Callable',
|
||||||
|
'ClassVar',
|
||||||
|
# ABCs (from collections.abc).
|
||||||
|
'AbstractSet', # collections.abc.Set.
|
||||||
|
'ByteString',
|
||||||
|
'Container',
|
||||||
|
# Concrete collection types.
|
||||||
|
'Counter',
|
||||||
|
'Deque',
|
||||||
|
'Dict',
|
||||||
|
'DefaultDict',
|
||||||
|
'List',
|
||||||
|
'Set',
|
||||||
|
'FrozenSet',
|
||||||
|
'NamedTuple', # Not really a type.
|
||||||
|
'Generator',
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def inline_comments_in_brackets_ruin_everything():
|
||||||
|
if typedargslist:
|
||||||
|
parameters.children = [
|
||||||
|
parameters.children[0], body, parameters.children[-1] # (1 # )1
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
parameters.children = [
|
||||||
|
parameters.children[0], # (2 what if this was actually long
|
||||||
|
body,
|
||||||
|
parameters.children[-1], # )2
|
||||||
|
]
|
||||||
|
if (
|
||||||
|
self._proc is not None and
|
||||||
|
# has the child process finished?
|
||||||
|
self._returncode is None and
|
||||||
|
# the child process has finished, but the
|
||||||
|
# transport hasn't been notified yet?
|
||||||
|
self._proc.poll() is None
|
||||||
|
):
|
||||||
|
pass
|
||||||
|
short = [
|
||||||
|
# one
|
||||||
|
1,
|
||||||
|
# two
|
||||||
|
2,
|
||||||
|
]
|
||||||
|
call(
|
||||||
|
arg1,
|
||||||
|
arg2,
|
||||||
|
"""
|
||||||
|
short
|
||||||
|
""",
|
||||||
|
arg3=True,
|
||||||
|
)
|
||||||
|
############################################################################
|
||||||
|
call2(
|
||||||
|
# short
|
||||||
|
arg1,
|
||||||
|
# but
|
||||||
|
arg2,
|
||||||
|
# multiline
|
||||||
|
"""
|
||||||
|
short
|
||||||
|
""",
|
||||||
|
# yup
|
||||||
|
arg3=True,
|
||||||
|
)
|
||||||
|
lcomp = [
|
||||||
|
element for element in collection if element is not None # yup # yup # right
|
||||||
|
]
|
||||||
|
lcomp2 = [
|
||||||
|
# hello
|
||||||
|
element
|
||||||
|
# yup
|
||||||
|
for element in collection
|
||||||
|
# right
|
||||||
|
if element is not None
|
||||||
|
]
|
||||||
|
lcomp3 = [
|
||||||
|
# This one is actually too long to fit in a single line.
|
||||||
|
element.split('\n', 1)[0]
|
||||||
|
# yup
|
||||||
|
for element in collection.select_elements()
|
||||||
|
# right
|
||||||
|
if element is not None
|
||||||
|
]
|
||||||
|
return Node(
|
||||||
|
syms.simple_stmt,
|
||||||
|
[Node(statement, result), Leaf(token.NEWLINE, '\n')], # FIXME: \r\n?
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
instruction()
|
||||||
|
# END COMMENTS
|
||||||
|
# MORE END COMMENTS
|
||||||
21
tests/composition.py
Normal file
21
tests/composition.py
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
class C:
|
||||||
|
|
||||||
|
def test(self) -> None:
|
||||||
|
with patch("black.out", print):
|
||||||
|
self.assertEqual(
|
||||||
|
unstyle(str(report)), '1 file reformatted, 1 file failed to reformat.'
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
unstyle(str(report)),
|
||||||
|
'1 file reformatted, 1 file left unchanged, 1 file failed to reformat.',
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
unstyle(str(report)),
|
||||||
|
'2 files reformatted, 1 file left unchanged, '
|
||||||
|
'1 file failed to reformat.',
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
unstyle(str(report)),
|
||||||
|
'2 files reformatted, 2 files left unchanged, '
|
||||||
|
'2 files failed to reformat.',
|
||||||
|
)
|
||||||
240
tests/expression.py
Normal file
240
tests/expression.py
Normal file
@ -0,0 +1,240 @@
|
|||||||
|
...
|
||||||
|
'some_string'
|
||||||
|
b'\\xa3'
|
||||||
|
Name
|
||||||
|
None
|
||||||
|
True
|
||||||
|
False
|
||||||
|
1
|
||||||
|
1.0
|
||||||
|
1j
|
||||||
|
True or False
|
||||||
|
True or False or None
|
||||||
|
True and False
|
||||||
|
True and False and None
|
||||||
|
(Name1 and Name2) or Name3
|
||||||
|
Name1 and Name2 or Name3
|
||||||
|
Name1 or (Name2 and Name3)
|
||||||
|
Name1 or Name2 and Name3
|
||||||
|
(Name1 and Name2) or (Name3 and Name4)
|
||||||
|
Name1 and Name2 or Name3 and Name4
|
||||||
|
Name1 or (Name2 and Name3) or Name4
|
||||||
|
Name1 or Name2 and Name3 or Name4
|
||||||
|
v1 << 2
|
||||||
|
1 >> v2
|
||||||
|
1 % finished
|
||||||
|
1 + v2 - v3 * 4 ^ 5 ** v6 / 7 // 8
|
||||||
|
((1 + v2) - (v3 * 4)) ^ (((5 ** v6) / 7) // 8)
|
||||||
|
not great
|
||||||
|
~great
|
||||||
|
+value
|
||||||
|
-1
|
||||||
|
~int and not v1 ^ 123 + v2 | True
|
||||||
|
(~int) and (not ((v1 ^ (123 + v2)) | True))
|
||||||
|
lambda arg: None
|
||||||
|
lambda a=True: a
|
||||||
|
lambda a, b, c=True: a
|
||||||
|
lambda a, b, c=True, *, d=(1 << v2), e='str': a
|
||||||
|
lambda a, b, c=True, *vararg, d=(v1 << 2), e='str', **kwargs: a + b
|
||||||
|
1 if True else 2
|
||||||
|
str or None if True else str or bytes or None
|
||||||
|
(str or None) if True else (str or bytes or None)
|
||||||
|
str or None if (1 if True else 2) else str or bytes or None
|
||||||
|
(str or None) if (1 if True else 2) else (str or bytes or None)
|
||||||
|
{'2.7': dead, '3.7': (long_live or die_hard)}
|
||||||
|
{'2.7': dead, '3.7': (long_live or die_hard), **{'3.6': verygood}}
|
||||||
|
{**a, **b, **c}
|
||||||
|
{'2.7', '3.6', '3.7', '3.8', '3.9', ('4.0' if gilectomy else '3.10')}
|
||||||
|
({'a': 'b'}, (True or False), (+value), 'string', b'bytes') or None
|
||||||
|
()
|
||||||
|
(1,)
|
||||||
|
(1, 2)
|
||||||
|
(1, 2, 3)
|
||||||
|
[]
|
||||||
|
[1, 2, 3, 4, 5, 6, 7, 8, 9, (10 or A), (11 or B), (12 or C)]
|
||||||
|
{i for i in (1, 2, 3)}
|
||||||
|
{(i ** 2) for i in (1, 2, 3)}
|
||||||
|
{(i ** 2) for i, _ in ((1, 'a'), (2, 'b'), (3, 'c'))}
|
||||||
|
{((i ** 2) + j) for i in (1, 2, 3) for j in (1, 2, 3)}
|
||||||
|
[i for i in (1, 2, 3)]
|
||||||
|
[(i ** 2) for i in (1, 2, 3)]
|
||||||
|
[(i ** 2) for i, _ in ((1, 'a'), (2, 'b'), (3, 'c'))]
|
||||||
|
[((i ** 2) + j) for i in (1, 2, 3) for j in (1, 2, 3)]
|
||||||
|
{i: 0 for i in (1, 2, 3)}
|
||||||
|
{i: j for i, j in ((1, 'a'), (2, 'b'), (3, 'c'))}
|
||||||
|
Python3 > Python2 > COBOL
|
||||||
|
Life is Life
|
||||||
|
call()
|
||||||
|
call(arg)
|
||||||
|
call(kwarg='hey')
|
||||||
|
call(arg, kwarg='hey')
|
||||||
|
call(arg, another, kwarg='hey', **kwargs)
|
||||||
|
lukasz.langa.pl
|
||||||
|
call.me(maybe)
|
||||||
|
1 .real
|
||||||
|
1.0 .real
|
||||||
|
....__class__
|
||||||
|
list[str]
|
||||||
|
dict[str, int]
|
||||||
|
tuple[str, ...]
|
||||||
|
tuple[str, int, float, dict[str, int]]
|
||||||
|
slice[0]
|
||||||
|
slice[0:1]
|
||||||
|
slice[0:1:2]
|
||||||
|
slice[:]
|
||||||
|
slice[:-1]
|
||||||
|
slice[1:]
|
||||||
|
slice[::-1]
|
||||||
|
(str or None) if (sys.version_info[0] > (3,)) else (str or bytes or None)
|
||||||
|
f'f-string without formatted values is just a string'
|
||||||
|
f'{{NOT a formatted value}}'
|
||||||
|
f'some f-string with {a} {few():.2f} {formatted.values!r}'
|
||||||
|
f"{f'{nested} inner'} outer"
|
||||||
|
f'space between opening braces: { {a for a in (1, 2, 3)}}'
|
||||||
|
{'2.7': dead, '3.7': long_live or die_hard}
|
||||||
|
{'2.7', '3.6', '3.7', '3.8', '3.9', '4.0' if gilectomy else '3.10'}
|
||||||
|
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10 or A, 11 or B, 12 or C]
|
||||||
|
(SomeName)
|
||||||
|
SomeName
|
||||||
|
(Good, Bad, Ugly)
|
||||||
|
(i for i in (1, 2, 3))
|
||||||
|
((i ** 2) for i in (1, 2, 3))
|
||||||
|
((i ** 2) for i, _ in ((1, 'a'), (2, 'b'), (3, 'c')))
|
||||||
|
(((i ** 2) + j) for i in (1, 2, 3) for j in (1, 2, 3))
|
||||||
|
(*starred)
|
||||||
|
a = (1,)
|
||||||
|
b = 1,
|
||||||
|
c = 1
|
||||||
|
d = (1,) + a + (2,)
|
||||||
|
|
||||||
|
|
||||||
|
def gen():
|
||||||
|
yield from outside_of_generator
|
||||||
|
a = (yield)
|
||||||
|
|
||||||
|
|
||||||
|
async def f():
|
||||||
|
await some.complicated[0].call(with_args=(True or (1 is not 1)))
|
||||||
|
|
||||||
|
|
||||||
|
# output
|
||||||
|
|
||||||
|
|
||||||
|
...
|
||||||
|
'some_string'
|
||||||
|
b'\\xa3'
|
||||||
|
Name
|
||||||
|
None
|
||||||
|
True
|
||||||
|
False
|
||||||
|
1
|
||||||
|
1.0
|
||||||
|
1j
|
||||||
|
True or False
|
||||||
|
True or False or None
|
||||||
|
True and False
|
||||||
|
True and False and None
|
||||||
|
(Name1 and Name2) or Name3
|
||||||
|
Name1 and Name2 or Name3
|
||||||
|
Name1 or (Name2 and Name3)
|
||||||
|
Name1 or Name2 and Name3
|
||||||
|
(Name1 and Name2) or (Name3 and Name4)
|
||||||
|
Name1 and Name2 or Name3 and Name4
|
||||||
|
Name1 or (Name2 and Name3) or Name4
|
||||||
|
Name1 or Name2 and Name3 or Name4
|
||||||
|
v1 << 2
|
||||||
|
1 >> v2
|
||||||
|
1 % finished
|
||||||
|
1 + v2 - v3 * 4 ^ 5 ** v6 / 7 // 8
|
||||||
|
((1 + v2) - (v3 * 4)) ^ (((5 ** v6) / 7) // 8)
|
||||||
|
not great
|
||||||
|
~great
|
||||||
|
+value
|
||||||
|
-1
|
||||||
|
~int and not v1 ^ 123 + v2 | True
|
||||||
|
(~int) and (not ((v1 ^ (123 + v2)) | True))
|
||||||
|
lambda arg: None
|
||||||
|
lambda a=True: a
|
||||||
|
lambda a, b, c=True: a
|
||||||
|
lambda a, b, c=True, *, d=(1 << v2), e='str': a
|
||||||
|
lambda a, b, c=True, *vararg, d=(v1 << 2), e='str', **kwargs: a + b
|
||||||
|
1 if True else 2
|
||||||
|
str or None if True else str or bytes or None
|
||||||
|
(str or None) if True else (str or bytes or None)
|
||||||
|
str or None if (1 if True else 2) else str or bytes or None
|
||||||
|
(str or None) if (1 if True else 2) else (str or bytes or None)
|
||||||
|
{'2.7': dead, '3.7': (long_live or die_hard)}
|
||||||
|
{'2.7': dead, '3.7': (long_live or die_hard), **{'3.6': verygood}}
|
||||||
|
{**a, **b, **c}
|
||||||
|
{'2.7', '3.6', '3.7', '3.8', '3.9', ('4.0' if gilectomy else '3.10')}
|
||||||
|
({'a': 'b'}, (True or False), (+value), 'string', b'bytes') or None
|
||||||
|
()
|
||||||
|
(1,)
|
||||||
|
(1, 2)
|
||||||
|
(1, 2, 3)
|
||||||
|
[]
|
||||||
|
[1, 2, 3, 4, 5, 6, 7, 8, 9, (10 or A), (11 or B), (12 or C)]
|
||||||
|
{i for i in (1, 2, 3)}
|
||||||
|
{(i ** 2) for i in (1, 2, 3)}
|
||||||
|
{(i ** 2) for i, _ in ((1, 'a'), (2, 'b'), (3, 'c'))}
|
||||||
|
{((i ** 2) + j) for i in (1, 2, 3) for j in (1, 2, 3)}
|
||||||
|
[i for i in (1, 2, 3)]
|
||||||
|
[(i ** 2) for i in (1, 2, 3)]
|
||||||
|
[(i ** 2) for i, _ in ((1, 'a'), (2, 'b'), (3, 'c'))]
|
||||||
|
[((i ** 2) + j) for i in (1, 2, 3) for j in (1, 2, 3)]
|
||||||
|
{i: 0 for i in (1, 2, 3)}
|
||||||
|
{i: j for i, j in ((1, 'a'), (2, 'b'), (3, 'c'))}
|
||||||
|
Python3 > Python2 > COBOL
|
||||||
|
Life is Life
|
||||||
|
call()
|
||||||
|
call(arg)
|
||||||
|
call(kwarg='hey')
|
||||||
|
call(arg, kwarg='hey')
|
||||||
|
call(arg, another, kwarg='hey', **kwargs)
|
||||||
|
lukasz.langa.pl
|
||||||
|
call.me(maybe)
|
||||||
|
1 .real
|
||||||
|
1.0 .real
|
||||||
|
....__class__
|
||||||
|
list[str]
|
||||||
|
dict[str, int]
|
||||||
|
tuple[str, ...]
|
||||||
|
tuple[str, int, float, dict[str, int]]
|
||||||
|
slice[0]
|
||||||
|
slice[0:1]
|
||||||
|
slice[0:1:2]
|
||||||
|
slice[:]
|
||||||
|
slice[:-1]
|
||||||
|
slice[1:]
|
||||||
|
slice[::-1]
|
||||||
|
(str or None) if (sys.version_info[0] > (3,)) else (str or bytes or None)
|
||||||
|
f'f-string without formatted values is just a string'
|
||||||
|
f'{{NOT a formatted value}}'
|
||||||
|
f'some f-string with {a} {few():.2f} {formatted.values!r}'
|
||||||
|
f"{f'{nested} inner'} outer"
|
||||||
|
f'space between opening braces: { {a for a in (1, 2, 3)}}'
|
||||||
|
{'2.7': dead, '3.7': long_live or die_hard}
|
||||||
|
{'2.7', '3.6', '3.7', '3.8', '3.9', '4.0' if gilectomy else '3.10'}
|
||||||
|
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10 or A, 11 or B, 12 or C]
|
||||||
|
(SomeName)
|
||||||
|
SomeName
|
||||||
|
(Good, Bad, Ugly)
|
||||||
|
(i for i in (1, 2, 3))
|
||||||
|
((i ** 2) for i in (1, 2, 3))
|
||||||
|
((i ** 2) for i, _ in ((1, 'a'), (2, 'b'), (3, 'c')))
|
||||||
|
(((i ** 2) + j) for i in (1, 2, 3) for j in (1, 2, 3))
|
||||||
|
(*starred)
|
||||||
|
a = (1,)
|
||||||
|
b = 1,
|
||||||
|
c = 1
|
||||||
|
d = (1,) + a + (2,)
|
||||||
|
|
||||||
|
|
||||||
|
def gen():
|
||||||
|
yield from outside_of_generator
|
||||||
|
|
||||||
|
a = (yield)
|
||||||
|
|
||||||
|
|
||||||
|
async def f():
|
||||||
|
await some.complicated[0].call(with_args=(True or (1 is not 1)))
|
||||||
137
tests/function.py
Normal file
137
tests/function.py
Normal file
@ -0,0 +1,137 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import asyncio
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from third_party import X, Y, Z
|
||||||
|
|
||||||
|
from library import some_connection, \
|
||||||
|
some_decorator
|
||||||
|
|
||||||
|
def func_no_args():
|
||||||
|
a; b; c
|
||||||
|
if True: raise RuntimeError
|
||||||
|
if False: ...
|
||||||
|
for i in range(10):
|
||||||
|
print(i)
|
||||||
|
continue
|
||||||
|
return None
|
||||||
|
async def coroutine(arg):
|
||||||
|
"Single-line docstring. Multiline is harder to reformat."
|
||||||
|
async with some_connection() as conn:
|
||||||
|
await conn.do_what_i_mean('SELECT bobby, tables FROM xkcd', timeout=2)
|
||||||
|
await asyncio.sleep(1)
|
||||||
|
@asyncio.coroutine
|
||||||
|
@some_decorator(
|
||||||
|
with_args=True,
|
||||||
|
many_args=[1,2,3]
|
||||||
|
)
|
||||||
|
def function_signature_stress_test(number:int,no_annotation=None,text:str="default",* ,debug:bool=False,**kwargs) -> str:
|
||||||
|
return text[number:-1]
|
||||||
|
|
||||||
|
def long_lines():
|
||||||
|
if True:
|
||||||
|
typedargslist.extend(
|
||||||
|
gen_annotated_params(ast_args.kwonlyargs, ast_args.kw_defaults, parameters, implicit_default=True)
|
||||||
|
)
|
||||||
|
_type_comment_re = re.compile(
|
||||||
|
r"""
|
||||||
|
^
|
||||||
|
[\t ]*
|
||||||
|
\#[ ]type:[ ]*
|
||||||
|
(?P<type>
|
||||||
|
[^#\t\n]+?
|
||||||
|
)
|
||||||
|
(?<!ignore) # note: this will force the non-greedy + in <type> to match
|
||||||
|
# a trailing space which is why we need the silliness below
|
||||||
|
(?<!ignore[ ]{1})(?<!ignore[ ]{2})(?<!ignore[ ]{3})(?<!ignore[ ]{4})
|
||||||
|
(?<!ignore[ ]{5})(?<!ignore[ ]{6})(?<!ignore[ ]{7})(?<!ignore[ ]{8})
|
||||||
|
(?<!ignore[ ]{9})(?<!ignore[ ]{10})
|
||||||
|
[\t ]*
|
||||||
|
(?P<nl>
|
||||||
|
(?:\#[^\n]*)?
|
||||||
|
\n?
|
||||||
|
)
|
||||||
|
$
|
||||||
|
""", re.MULTILINE | re.VERBOSE
|
||||||
|
)
|
||||||
|
|
||||||
|
# output
|
||||||
|
|
||||||
|
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import asyncio
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from third_party import X, Y, Z
|
||||||
|
|
||||||
|
from library import some_connection, some_decorator
|
||||||
|
|
||||||
|
|
||||||
|
def func_no_args():
|
||||||
|
a
|
||||||
|
b
|
||||||
|
c
|
||||||
|
if True:
|
||||||
|
raise RuntimeError
|
||||||
|
|
||||||
|
if False:
|
||||||
|
...
|
||||||
|
for i in range(10):
|
||||||
|
print(i)
|
||||||
|
continue
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def coroutine(arg):
|
||||||
|
"Single-line docstring. Multiline is harder to reformat."
|
||||||
|
async with some_connection() as conn:
|
||||||
|
await conn.do_what_i_mean('SELECT bobby, tables FROM xkcd', timeout=2)
|
||||||
|
await asyncio.sleep(1)
|
||||||
|
|
||||||
|
|
||||||
|
@asyncio.coroutine
|
||||||
|
@some_decorator(with_args=True, many_args=[1, 2, 3])
|
||||||
|
def function_signature_stress_test(
|
||||||
|
number: int,
|
||||||
|
no_annotation=None,
|
||||||
|
text: str = "default",
|
||||||
|
*,
|
||||||
|
debug: bool = False,
|
||||||
|
**kwargs,
|
||||||
|
) -> str:
|
||||||
|
return text[number:-1]
|
||||||
|
|
||||||
|
|
||||||
|
def long_lines():
|
||||||
|
if True:
|
||||||
|
typedargslist.extend(
|
||||||
|
gen_annotated_params(
|
||||||
|
ast_args.kwonlyargs,
|
||||||
|
ast_args.kw_defaults,
|
||||||
|
parameters,
|
||||||
|
implicit_default=True,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
_type_comment_re = re.compile(
|
||||||
|
r"""
|
||||||
|
^
|
||||||
|
[\t ]*
|
||||||
|
\#[ ]type:[ ]*
|
||||||
|
(?P<type>
|
||||||
|
[^#\t\n]+?
|
||||||
|
)
|
||||||
|
(?<!ignore) # note: this will force the non-greedy + in <type> to match
|
||||||
|
# a trailing space which is why we need the silliness below
|
||||||
|
(?<!ignore[ ]{1})(?<!ignore[ ]{2})(?<!ignore[ ]{3})(?<!ignore[ ]{4})
|
||||||
|
(?<!ignore[ ]{5})(?<!ignore[ ]{6})(?<!ignore[ ]{7})(?<!ignore[ ]{8})
|
||||||
|
(?<!ignore[ ]{9})(?<!ignore[ ]{10})
|
||||||
|
[\t ]*
|
||||||
|
(?P<nl>
|
||||||
|
(?:\#[^\n]*)?
|
||||||
|
\n?
|
||||||
|
)
|
||||||
|
$
|
||||||
|
""",
|
||||||
|
re.MULTILINE | re.VERBOSE,
|
||||||
|
)
|
||||||
77
tests/import_spacing.py
Normal file
77
tests/import_spacing.py
Normal file
@ -0,0 +1,77 @@
|
|||||||
|
"""The asyncio package, tracking PEP 3156."""
|
||||||
|
|
||||||
|
# flake8: noqa
|
||||||
|
|
||||||
|
import sys
|
||||||
|
|
||||||
|
# This relies on each of the submodules having an __all__ variable.
|
||||||
|
from .base_events import *
|
||||||
|
from .coroutines import *
|
||||||
|
from .events import * # comment here
|
||||||
|
|
||||||
|
from .futures import *
|
||||||
|
from .locks import * # comment here
|
||||||
|
from .protocols import *
|
||||||
|
|
||||||
|
from .runners import * # comment here
|
||||||
|
from .queues import *
|
||||||
|
from .streams import *
|
||||||
|
|
||||||
|
from .subprocess import *
|
||||||
|
from .tasks import *
|
||||||
|
from .transports import *
|
||||||
|
|
||||||
|
__all__ = (
|
||||||
|
base_events.__all__ +
|
||||||
|
coroutines.__all__ +
|
||||||
|
events.__all__ +
|
||||||
|
futures.__all__ +
|
||||||
|
locks.__all__ +
|
||||||
|
protocols.__all__ +
|
||||||
|
runners.__all__ +
|
||||||
|
queues.__all__ +
|
||||||
|
streams.__all__ +
|
||||||
|
subprocess.__all__ +
|
||||||
|
tasks.__all__ +
|
||||||
|
transports.__all__
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# output
|
||||||
|
|
||||||
|
|
||||||
|
"""The asyncio package, tracking PEP 3156."""
|
||||||
|
# flake8: noqa
|
||||||
|
import sys
|
||||||
|
|
||||||
|
# This relies on each of the submodules having an __all__ variable.
|
||||||
|
from .base_events import *
|
||||||
|
from .coroutines import *
|
||||||
|
from .events import * # comment here
|
||||||
|
|
||||||
|
from .futures import *
|
||||||
|
from .locks import * # comment here
|
||||||
|
from .protocols import *
|
||||||
|
|
||||||
|
from .runners import * # comment here
|
||||||
|
from .queues import *
|
||||||
|
from .streams import *
|
||||||
|
|
||||||
|
from .subprocess import *
|
||||||
|
from .tasks import *
|
||||||
|
from .transports import *
|
||||||
|
|
||||||
|
__all__ = (
|
||||||
|
base_events.__all__ +
|
||||||
|
coroutines.__all__ +
|
||||||
|
events.__all__ +
|
||||||
|
futures.__all__ +
|
||||||
|
locks.__all__ +
|
||||||
|
protocols.__all__ +
|
||||||
|
runners.__all__ +
|
||||||
|
queues.__all__ +
|
||||||
|
streams.__all__ +
|
||||||
|
subprocess.__all__ +
|
||||||
|
tasks.__all__ +
|
||||||
|
transports.__all__
|
||||||
|
)
|
||||||
220
tests/test_black.py
Normal file
220
tests/test_black.py
Normal file
@ -0,0 +1,220 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
from functools import partial
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import List, Tuple
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from click import unstyle
|
||||||
|
|
||||||
|
import black
|
||||||
|
|
||||||
|
ll = 88
|
||||||
|
ff = partial(black.format_file, line_length=ll, fast=True)
|
||||||
|
fs = partial(black.format_str, line_length=ll)
|
||||||
|
THIS_FILE = Path(__file__)
|
||||||
|
THIS_DIR = THIS_FILE.parent
|
||||||
|
|
||||||
|
|
||||||
|
def dump_to_stderr(*output: str) -> str:
|
||||||
|
return '\n' + '\n'.join(output) + '\n'
|
||||||
|
|
||||||
|
|
||||||
|
def read_data(name: str) -> Tuple[str, str]:
|
||||||
|
"""read_data('test_name') -> 'input', 'output'"""
|
||||||
|
if not name.endswith('.py'):
|
||||||
|
name += '.py'
|
||||||
|
_input: List[str] = []
|
||||||
|
_output: List[str] = []
|
||||||
|
with open(THIS_DIR / name, 'r', encoding='utf8') as test:
|
||||||
|
lines = test.readlines()
|
||||||
|
result = _input
|
||||||
|
for line in lines:
|
||||||
|
if line.rstrip() == '# output':
|
||||||
|
result = _output
|
||||||
|
continue
|
||||||
|
|
||||||
|
result.append(line)
|
||||||
|
if _input and not _output:
|
||||||
|
# If there's no output marker, treat the entire file as already pre-formatted.
|
||||||
|
_output = _input[:]
|
||||||
|
return ''.join(_input).strip() + '\n', ''.join(_output).strip() + '\n'
|
||||||
|
|
||||||
|
|
||||||
|
class BlackTestCase(unittest.TestCase):
|
||||||
|
maxDiff = None
|
||||||
|
|
||||||
|
def assertFormatEqual(self, expected: str, actual: str) -> None:
|
||||||
|
if actual != expected:
|
||||||
|
black.out('Expected tree:', fg='green')
|
||||||
|
try:
|
||||||
|
exp_node = black.lib2to3_parse(expected)
|
||||||
|
bdv = black.DebugVisitor()
|
||||||
|
list(bdv.visit(exp_node))
|
||||||
|
except Exception as ve:
|
||||||
|
black.err(str(ve))
|
||||||
|
black.out('Actual tree:', fg='red')
|
||||||
|
try:
|
||||||
|
exp_node = black.lib2to3_parse(actual)
|
||||||
|
bdv = black.DebugVisitor()
|
||||||
|
list(bdv.visit(exp_node))
|
||||||
|
except Exception as ve:
|
||||||
|
black.err(str(ve))
|
||||||
|
self.assertEqual(expected, actual)
|
||||||
|
|
||||||
|
@patch("black.dump_to_file", dump_to_stderr)
|
||||||
|
def test_self(self) -> None:
|
||||||
|
source, expected = read_data('test_black')
|
||||||
|
actual = fs(source)
|
||||||
|
self.assertFormatEqual(expected, actual)
|
||||||
|
black.assert_equivalent(source, actual)
|
||||||
|
black.assert_stable(source, actual, line_length=ll)
|
||||||
|
with self.assertRaises(black.NothingChanged):
|
||||||
|
ff(THIS_FILE)
|
||||||
|
|
||||||
|
@patch("black.dump_to_file", dump_to_stderr)
|
||||||
|
def test_black(self) -> None:
|
||||||
|
source, expected = read_data('../black')
|
||||||
|
actual = fs(source)
|
||||||
|
self.assertFormatEqual(expected, actual)
|
||||||
|
black.assert_equivalent(source, actual)
|
||||||
|
black.assert_stable(source, actual, line_length=ll)
|
||||||
|
with self.assertRaises(black.NothingChanged):
|
||||||
|
ff(THIS_FILE)
|
||||||
|
|
||||||
|
@patch("black.dump_to_file", dump_to_stderr)
|
||||||
|
def test_setup(self) -> None:
|
||||||
|
source, expected = read_data('../setup')
|
||||||
|
actual = fs(source)
|
||||||
|
self.assertFormatEqual(expected, actual)
|
||||||
|
black.assert_equivalent(source, actual)
|
||||||
|
black.assert_stable(source, actual, line_length=ll)
|
||||||
|
with self.assertRaises(black.NothingChanged):
|
||||||
|
ff(THIS_FILE)
|
||||||
|
|
||||||
|
@patch("black.dump_to_file", dump_to_stderr)
|
||||||
|
def test_function(self) -> None:
|
||||||
|
source, expected = read_data('function')
|
||||||
|
actual = fs(source)
|
||||||
|
self.assertFormatEqual(expected, actual)
|
||||||
|
black.assert_equivalent(source, actual)
|
||||||
|
black.assert_stable(source, actual, line_length=ll)
|
||||||
|
|
||||||
|
@patch("black.dump_to_file", dump_to_stderr)
|
||||||
|
def test_expression(self) -> None:
|
||||||
|
source, expected = read_data('expression')
|
||||||
|
actual = fs(source)
|
||||||
|
self.assertFormatEqual(expected, actual)
|
||||||
|
black.assert_equivalent(source, actual)
|
||||||
|
black.assert_stable(source, actual, line_length=ll)
|
||||||
|
|
||||||
|
@patch("black.dump_to_file", dump_to_stderr)
|
||||||
|
def test_comments(self) -> None:
|
||||||
|
source, expected = read_data('comments')
|
||||||
|
actual = fs(source)
|
||||||
|
self.assertFormatEqual(expected, actual)
|
||||||
|
black.assert_equivalent(source, actual)
|
||||||
|
black.assert_stable(source, actual, line_length=ll)
|
||||||
|
|
||||||
|
@patch("black.dump_to_file", dump_to_stderr)
|
||||||
|
def test_comments2(self) -> None:
|
||||||
|
source, expected = read_data('comments2')
|
||||||
|
actual = fs(source)
|
||||||
|
self.assertFormatEqual(expected, actual)
|
||||||
|
black.assert_equivalent(source, actual)
|
||||||
|
black.assert_stable(source, actual, line_length=ll)
|
||||||
|
|
||||||
|
@patch("black.dump_to_file", dump_to_stderr)
|
||||||
|
def test_cantfit(self) -> None:
|
||||||
|
source, expected = read_data('cantfit')
|
||||||
|
actual = fs(source)
|
||||||
|
self.assertFormatEqual(expected, actual)
|
||||||
|
black.assert_equivalent(source, actual)
|
||||||
|
black.assert_stable(source, actual, line_length=ll)
|
||||||
|
|
||||||
|
@patch("black.dump_to_file", dump_to_stderr)
|
||||||
|
def test_import_spacing(self) -> None:
|
||||||
|
source, expected = read_data('import_spacing')
|
||||||
|
actual = fs(source)
|
||||||
|
self.assertFormatEqual(expected, actual)
|
||||||
|
black.assert_equivalent(source, actual)
|
||||||
|
black.assert_stable(source, actual, line_length=ll)
|
||||||
|
|
||||||
|
@patch("black.dump_to_file", dump_to_stderr)
|
||||||
|
def test_composition(self) -> None:
|
||||||
|
source, expected = read_data('composition')
|
||||||
|
actual = fs(source)
|
||||||
|
self.assertFormatEqual(expected, actual)
|
||||||
|
black.assert_equivalent(source, actual)
|
||||||
|
black.assert_stable(source, actual, line_length=ll)
|
||||||
|
|
||||||
|
def test_report(self) -> None:
|
||||||
|
report = black.Report()
|
||||||
|
out_lines = []
|
||||||
|
err_lines = []
|
||||||
|
|
||||||
|
def out(msg: str, **kwargs):
|
||||||
|
out_lines.append(msg)
|
||||||
|
|
||||||
|
def err(msg: str, **kwargs):
|
||||||
|
err_lines.append(msg)
|
||||||
|
|
||||||
|
with patch("black.out", out), patch("black.err", err):
|
||||||
|
report.done(Path('f1'), changed=True)
|
||||||
|
self.assertEqual(len(out_lines), 1)
|
||||||
|
self.assertEqual(len(err_lines), 0)
|
||||||
|
self.assertEqual(out_lines[-1], 'reformatted f1')
|
||||||
|
self.assertEqual(unstyle(str(report)), '1 file reformatted.')
|
||||||
|
self.assertEqual(report.return_code, 0)
|
||||||
|
report.failed(Path('e1'), 'boom')
|
||||||
|
self.assertEqual(len(out_lines), 1)
|
||||||
|
self.assertEqual(len(err_lines), 1)
|
||||||
|
self.assertEqual(err_lines[-1], 'error: cannot format e1: boom')
|
||||||
|
self.assertEqual(
|
||||||
|
unstyle(str(report)), '1 file reformatted, 1 file failed to reformat.'
|
||||||
|
)
|
||||||
|
self.assertEqual(report.return_code, 1)
|
||||||
|
report.done(Path('f2'), changed=False)
|
||||||
|
self.assertEqual(len(out_lines), 2)
|
||||||
|
self.assertEqual(len(err_lines), 1)
|
||||||
|
self.assertEqual(out_lines[-1], 'f2 already well formatted, good job.')
|
||||||
|
self.assertEqual(
|
||||||
|
unstyle(str(report)),
|
||||||
|
'1 file reformatted, 1 file left unchanged, '
|
||||||
|
'1 file failed to reformat.',
|
||||||
|
)
|
||||||
|
self.assertEqual(report.return_code, 1)
|
||||||
|
report.done(Path('f3'), changed=True)
|
||||||
|
self.assertEqual(len(out_lines), 3)
|
||||||
|
self.assertEqual(len(err_lines), 1)
|
||||||
|
self.assertEqual(out_lines[-1], 'reformatted f3')
|
||||||
|
self.assertEqual(
|
||||||
|
unstyle(str(report)),
|
||||||
|
'2 files reformatted, 1 file left unchanged, '
|
||||||
|
'1 file failed to reformat.',
|
||||||
|
)
|
||||||
|
self.assertEqual(report.return_code, 1)
|
||||||
|
report.failed(Path('e2'), 'boom')
|
||||||
|
self.assertEqual(len(out_lines), 3)
|
||||||
|
self.assertEqual(len(err_lines), 2)
|
||||||
|
self.assertEqual(err_lines[-1], 'error: cannot format e2: boom')
|
||||||
|
self.assertEqual(
|
||||||
|
unstyle(str(report)),
|
||||||
|
'2 files reformatted, 1 file left unchanged, '
|
||||||
|
'2 files failed to reformat.',
|
||||||
|
)
|
||||||
|
self.assertEqual(report.return_code, 1)
|
||||||
|
report.done(Path('f4'), changed=False)
|
||||||
|
self.assertEqual(len(out_lines), 4)
|
||||||
|
self.assertEqual(len(err_lines), 2)
|
||||||
|
self.assertEqual(out_lines[-1], 'f4 already well formatted, good job.')
|
||||||
|
self.assertEqual(
|
||||||
|
unstyle(str(report)),
|
||||||
|
'2 files reformatted, 2 files left unchanged, '
|
||||||
|
'2 files failed to reformat.',
|
||||||
|
)
|
||||||
|
self.assertEqual(report.return_code, 1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
Loading…
Reference in New Issue
Block a user